From 547304f2b4b5aa1f73e7ad985a2e5abeaa9d8678 Mon Sep 17 00:00:00 2001 From: agobrech Date: Thu, 23 Feb 2023 17:13:16 +0100 Subject: [PATCH 1/7] Remove brackets and double quotes from fieldname --- .../nodes/ItemLists/summarize.operation.ts | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/packages/nodes-base/nodes/ItemLists/summarize.operation.ts b/packages/nodes-base/nodes/ItemLists/summarize.operation.ts index 24d8f6fc6a27b..b7ded65dc24fe 100644 --- a/packages/nodes-base/nodes/ItemLists/summarize.operation.ts +++ b/packages/nodes-base/nodes/ItemLists/summarize.operation.ts @@ -320,6 +320,25 @@ function isEmpty(value: T) { return value === undefined || value === null || value === ''; } +function parseFieldName(returnData: IDataObject) { + console.log('\n First', returnData); + const regexBrackets = /[\]\["]/g; + const regexSpaces = /[ .]/g; + for (const key of Object.keys(returnData)) { + if (key.match(regexBrackets)) { + const newKey = key.replace(regexBrackets, ''); + returnData[newKey] = returnData[key]; + delete returnData[key]; + } + if (key.match(regexSpaces)) { + const newKey = key.replace(regexSpaces, '_'); + returnData[newKey] = returnData[key]; + delete returnData[key]; + } + } + console.log('After', returnData); +} + const fieldValueGetter = (disableDotNotation?: boolean) => { if (disableDotNotation) { return (item: IDataObject, field: string) => item[field]; @@ -432,7 +451,9 @@ function aggregateData( ); return acc; }, {} as IDataObject); + parseFieldName(returnData); if (options.outputFormat === 'singleItem') { + parseFieldName(returnData); return returnData; } else { return { ...returnData, pairedItems: data.map((item) => item._itemIndex as number) }; From b5faa6da06009bdf90e6e0c992a403f24e6d8c2e Mon Sep 17 00:00:00 2001 From: agobrech Date: Thu, 23 Feb 2023 17:33:14 +0100 Subject: [PATCH 2/7] Fix bug with duplicate field --- packages/nodes-base/nodes/ItemLists/summarize.operation.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/nodes-base/nodes/ItemLists/summarize.operation.ts b/packages/nodes-base/nodes/ItemLists/summarize.operation.ts index b7ded65dc24fe..02d6cf2c0f64e 100644 --- a/packages/nodes-base/nodes/ItemLists/summarize.operation.ts +++ b/packages/nodes-base/nodes/ItemLists/summarize.operation.ts @@ -330,6 +330,8 @@ function parseFieldName(returnData: IDataObject) { returnData[newKey] = returnData[key]; delete returnData[key]; } + } + for (const key of Object.keys(returnData)) { if (key.match(regexSpaces)) { const newKey = key.replace(regexSpaces, '_'); returnData[newKey] = returnData[key]; From 42cfe817cee680b398e4a4cb513e21a5ccf0b300 Mon Sep 17 00:00:00 2001 From: agobrech Date: Thu, 23 Feb 2023 18:20:33 +0100 Subject: [PATCH 3/7] Parse field names from splitbyfield --- .../nodes/ItemLists/summarize.operation.ts | 22 ++++++++++++++----- 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/packages/nodes-base/nodes/ItemLists/summarize.operation.ts b/packages/nodes-base/nodes/ItemLists/summarize.operation.ts index 02d6cf2c0f64e..919aba3cd3406 100644 --- a/packages/nodes-base/nodes/ItemLists/summarize.operation.ts +++ b/packages/nodes-base/nodes/ItemLists/summarize.operation.ts @@ -320,8 +320,8 @@ function isEmpty(value: T) { return value === undefined || value === null || value === ''; } -function parseFieldName(returnData: IDataObject) { - console.log('\n First', returnData); +function parseReturnData(returnData: IDataObject) { + //console.log('\n First', returnData); const regexBrackets = /[\]\["]/g; const regexSpaces = /[ .]/g; for (const key of Object.keys(returnData)) { @@ -338,7 +338,17 @@ function parseFieldName(returnData: IDataObject) { delete returnData[key]; } } - console.log('After', returnData); + //console.log('After', returnData); +} + +function parseFieldName(fieldName: string[]) { + const regexBrackets = /[\]\["]/g; + const regexSpaces = /[ .]/g; + fieldName.map((field) => { + field = field.replace(regexBrackets, ''); + field = field.replace(regexSpaces, '_'); + return field; + }); } const fieldValueGetter = (disableDotNotation?: boolean) => { @@ -453,9 +463,9 @@ function aggregateData( ); return acc; }, {} as IDataObject); - parseFieldName(returnData); + parseReturnData(returnData); if (options.outputFormat === 'singleItem') { - parseFieldName(returnData); + parseReturnData(returnData); return returnData; } else { return { ...returnData, pairedItems: data.map((item) => item._itemIndex as number) }; @@ -507,7 +517,7 @@ function aggregationToArray( previousStage: IDataObject = {}, ) { const returnData: IDataObject[] = []; - + parseFieldName(fieldsToSplitBy); const splitFieldName = fieldsToSplitBy[0]; const isNext = fieldsToSplitBy[1]; From 80f694f238ab26e2b12aee6bdb397f6e9ac76e20 Mon Sep 17 00:00:00 2001 From: agobrech Date: Mon, 27 Feb 2023 12:34:26 +0100 Subject: [PATCH 4/7] Fix error with field name remove console.logs --- packages/nodes-base/nodes/ItemLists/summarize.operation.ts | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/packages/nodes-base/nodes/ItemLists/summarize.operation.ts b/packages/nodes-base/nodes/ItemLists/summarize.operation.ts index 919aba3cd3406..3ba0632569ab5 100644 --- a/packages/nodes-base/nodes/ItemLists/summarize.operation.ts +++ b/packages/nodes-base/nodes/ItemLists/summarize.operation.ts @@ -321,7 +321,6 @@ function isEmpty(value: T) { } function parseReturnData(returnData: IDataObject) { - //console.log('\n First', returnData); const regexBrackets = /[\]\["]/g; const regexSpaces = /[ .]/g; for (const key of Object.keys(returnData)) { @@ -338,17 +337,17 @@ function parseReturnData(returnData: IDataObject) { delete returnData[key]; } } - //console.log('After', returnData); } function parseFieldName(fieldName: string[]) { const regexBrackets = /[\]\["]/g; const regexSpaces = /[ .]/g; - fieldName.map((field) => { + fieldName = fieldName.map((field) => { field = field.replace(regexBrackets, ''); field = field.replace(regexSpaces, '_'); return field; }); + return fieldName; } const fieldValueGetter = (disableDotNotation?: boolean) => { @@ -517,7 +516,7 @@ function aggregationToArray( previousStage: IDataObject = {}, ) { const returnData: IDataObject[] = []; - parseFieldName(fieldsToSplitBy); + fieldsToSplitBy = parseFieldName(fieldsToSplitBy); const splitFieldName = fieldsToSplitBy[0]; const isNext = fieldsToSplitBy[1]; From 89e7d177a17f428414c9306cbe47ecb83dbd142a Mon Sep 17 00:00:00 2001 From: agobrech Date: Tue, 28 Feb 2023 15:08:40 +0100 Subject: [PATCH 5/7] Add versioning to itemlist --- .../nodes/ItemLists/ItemList.node.ts | 27 + .../ItemListsV1.node.ts} | 1330 +++++++-------- .../nodes/ItemLists/V1/summarize.operation.ts | 582 +++++++ .../nodes/ItemLists/V2/ItemListsV2.node.ts | 1428 +++++++++++++++++ .../ItemLists/{ => V2}/summarize.operation.ts | 0 5 files changed, 2702 insertions(+), 665 deletions(-) create mode 100644 packages/nodes-base/nodes/ItemLists/ItemList.node.ts rename packages/nodes-base/nodes/ItemLists/{ItemLists.node.ts => V1/ItemListsV1.node.ts} (55%) create mode 100644 packages/nodes-base/nodes/ItemLists/V1/summarize.operation.ts create mode 100644 packages/nodes-base/nodes/ItemLists/V2/ItemListsV2.node.ts rename packages/nodes-base/nodes/ItemLists/{ => V2}/summarize.operation.ts (100%) diff --git a/packages/nodes-base/nodes/ItemLists/ItemList.node.ts b/packages/nodes-base/nodes/ItemLists/ItemList.node.ts new file mode 100644 index 0000000000000..542186e71b956 --- /dev/null +++ b/packages/nodes-base/nodes/ItemLists/ItemList.node.ts @@ -0,0 +1,27 @@ +import type { INodeTypeBaseDescription, IVersionedNodeType } from 'n8n-workflow'; +import { VersionedNodeType } from 'n8n-workflow'; + +import { ItemListsV1 } from './V1/ItemListsV1.node'; + +import { ItemListsV2 } from './V2/ItemListsV2.node'; + +export class ItemList extends VersionedNodeType { + constructor() { + const baseDescription: INodeTypeBaseDescription = { + displayName: 'Item Lists', + name: 'itemLists', + icon: 'file:itemLists.svg', + group: ['input'], + subtitle: '={{$parameter["operation"] + ": " + $parameter["resource"]}}', + description: 'Helper for working with lists of items and transforming arrays', + defaultVersion: 2, + }; + + const nodeVersions: IVersionedNodeType['nodeVersions'] = { + 1: new ItemListsV1(baseDescription), + 2: new ItemListsV2(baseDescription), + }; + + super(nodeVersions, baseDescription); + } +} diff --git a/packages/nodes-base/nodes/ItemLists/ItemLists.node.ts b/packages/nodes-base/nodes/ItemLists/V1/ItemListsV1.node.ts similarity index 55% rename from packages/nodes-base/nodes/ItemLists/ItemLists.node.ts rename to packages/nodes-base/nodes/ItemLists/V1/ItemListsV1.node.ts index 11011680f1afb..7e230c94d17c0 100644 --- a/packages/nodes-base/nodes/ItemLists/ItemLists.node.ts +++ b/packages/nodes-base/nodes/ItemLists/V1/ItemListsV1.node.ts @@ -7,6 +7,7 @@ import type { INode, INodeExecutionData, INodeType, + INodeTypeBaseDescription, INodeTypeDescription, } from 'n8n-workflow'; import { NodeOperationError } from 'n8n-workflow'; @@ -61,548 +62,546 @@ const shuffleArray = (array: any[]) => { import * as summarize from './summarize.operation'; -export class ItemLists implements INodeType { - description: INodeTypeDescription = { - displayName: 'Item Lists', - name: 'itemLists', - icon: 'file:itemLists.svg', - group: ['input'], - version: 1, - subtitle: '={{$parameter["operation"] + ": " + $parameter["resource"]}}', - description: 'Helper for working with lists of items and transforming arrays', - defaults: { - name: 'Item Lists', - }, - inputs: ['main'], - outputs: ['main'], - credentials: [], - properties: [ - { - displayName: 'Resource', - name: 'resource', - type: 'hidden', - options: [ - { - name: 'Item List', - value: 'itemList', - }, - ], - default: 'itemList', - }, - { - displayName: 'Operation', - name: 'operation', - type: 'options', - noDataExpression: true, - options: [ - { - name: 'Concatenate Items', - value: 'aggregateItems', - description: 'Combine fields into a list in a single new item', - action: 'Combine fields into a list in a single new item', - }, - { - name: 'Limit', - value: 'limit', - description: 'Remove items if there are too many', - action: 'Remove items if there are too many', - }, - { - name: 'Remove Duplicates', - value: 'removeDuplicates', - description: 'Remove extra items that are similar', - action: 'Remove extra items that are similar', - }, - { - name: 'Sort', - value: 'sort', - description: 'Change the item order', - action: 'Change the item order', - }, - { - name: 'Split Out Items', - value: 'splitOutItems', - description: 'Turn a list inside item(s) into separate items', - action: 'Turn a list inside item(s) into separate items', - }, - { - name: 'Summarize', - value: 'summarize', - description: 'Aggregate items together (pivot table)', - action: 'Aggregate items together (pivot table)', - }, - ], - default: 'splitOutItems', - }, - // Split out items - Fields - { - displayName: 'Field To Split Out', - name: 'fieldToSplitOut', - type: 'string', - default: '', - required: true, - displayOptions: { - show: { - resource: ['itemList'], - operation: ['splitOutItems'], - }, - }, - description: 'The name of the input field to break out into separate items', - requiresDataPath: 'single', +export class ItemListsV1 implements INodeType { + description: INodeTypeDescription; + + constructor(baseDescription: INodeTypeBaseDescription) { + this.description = { + ...baseDescription, + version: 1, + defaults: { + name: 'Item Lists', }, - { - displayName: 'Include', - name: 'include', - type: 'options', - options: [ - { - name: 'No Other Fields', - value: 'noOtherFields', - }, - { - name: 'All Other Fields', - value: 'allOtherFields', - }, - { - name: 'Selected Other Fields', - value: 'selectedOtherFields', - }, - ], - default: 'noOtherFields', - description: 'Whether to copy any other fields into the new items', - displayOptions: { - show: { - resource: ['itemList'], - operation: ['splitOutItems'], - }, + inputs: ['main'], + outputs: ['main'], + credentials: [], + properties: [ + { + displayName: 'Resource', + name: 'resource', + type: 'hidden', + options: [ + { + name: 'Item List', + value: 'itemList', + }, + ], + default: 'itemList', }, - }, - { - displayName: 'Fields To Include', - name: 'fieldsToInclude', - type: 'fixedCollection', - typeOptions: { - multipleValues: true, + { + displayName: 'Operation', + name: 'operation', + type: 'options', + noDataExpression: true, + options: [ + { + name: 'Concatenate Items', + value: 'aggregateItems', + description: 'Combine fields into a list in a single new item', + action: 'Combine fields into a list in a single new item', + }, + { + name: 'Limit', + value: 'limit', + description: 'Remove items if there are too many', + action: 'Remove items if there are too many', + }, + { + name: 'Remove Duplicates', + value: 'removeDuplicates', + description: 'Remove extra items that are similar', + action: 'Remove extra items that are similar', + }, + { + name: 'Sort', + value: 'sort', + description: 'Change the item order', + action: 'Change the item order', + }, + { + name: 'Split Out Items', + value: 'splitOutItems', + description: 'Turn a list inside item(s) into separate items', + action: 'Turn a list inside item(s) into separate items', + }, + { + name: 'Summarize', + value: 'summarize', + description: 'Aggregate items together (pivot table)', + action: 'Aggregate items together (pivot table)', + }, + ], + default: 'splitOutItems', }, - placeholder: 'Add Field To Include', - default: {}, - displayOptions: { - show: { - resource: ['itemList'], - operation: ['splitOutItems'], - include: ['selectedOtherFields'], + // Split out items - Fields + { + displayName: 'Field To Split Out', + name: 'fieldToSplitOut', + type: 'string', + default: '', + required: true, + displayOptions: { + show: { + resource: ['itemList'], + operation: ['splitOutItems'], + }, }, + description: 'The name of the input field to break out into separate items', + requiresDataPath: 'single', }, - options: [ - { - displayName: '', - name: 'fields', - values: [ - { - displayName: 'Field Name', - name: 'fieldName', - type: 'string', - default: '', - description: 'A field in the input items to aggregate together', - // eslint-disable-next-line n8n-nodes-base/node-param-placeholder-miscased-id - placeholder: 'e.g. id', - hint: ' Enter the field name as text', - requiresDataPath: 'single', - }, - ], - }, - ], - }, - // Aggregate Items - { - displayName: 'Aggregate', - name: 'aggregate', - type: 'options', - default: 'aggregateIndividualFields', - options: [ - { - name: 'Individual Fields', - value: 'aggregateIndividualFields', + { + displayName: 'Include', + name: 'include', + type: 'options', + options: [ + { + name: 'No Other Fields', + value: 'noOtherFields', + }, + { + name: 'All Other Fields', + value: 'allOtherFields', + }, + { + name: 'Selected Other Fields', + value: 'selectedOtherFields', + }, + ], + default: 'noOtherFields', + description: 'Whether to copy any other fields into the new items', + displayOptions: { + show: { + resource: ['itemList'], + operation: ['splitOutItems'], + }, }, - { - name: 'All Item Data (Into a Single List)', - value: 'aggregateAllItemData', + }, + { + displayName: 'Fields To Include', + name: 'fieldsToInclude', + type: 'fixedCollection', + typeOptions: { + multipleValues: true, }, - ], - displayOptions: { - show: { - resource: ['itemList'], - operation: ['aggregateItems'], + placeholder: 'Add Field To Include', + default: {}, + displayOptions: { + show: { + resource: ['itemList'], + operation: ['splitOutItems'], + include: ['selectedOtherFields'], + }, }, + options: [ + { + displayName: '', + name: 'fields', + values: [ + { + displayName: 'Field Name', + name: 'fieldName', + type: 'string', + default: '', + description: 'A field in the input items to aggregate together', + // eslint-disable-next-line n8n-nodes-base/node-param-placeholder-miscased-id + placeholder: 'e.g. id', + hint: ' Enter the field name as text', + requiresDataPath: 'single', + }, + ], + }, + ], }, - }, - // Aggregate Individual Fields - { - displayName: 'Fields To Aggregate', - name: 'fieldsToAggregate', - type: 'fixedCollection', - typeOptions: { - multipleValues: true, - }, - placeholder: 'Add Field To Aggregate', - default: { fieldToAggregate: [{ fieldToAggregate: '', renameField: false }] }, - displayOptions: { - show: { - resource: ['itemList'], - operation: ['aggregateItems'], - aggregate: ['aggregateIndividualFields'], + // Aggregate Items + { + displayName: 'Aggregate', + name: 'aggregate', + type: 'options', + default: 'aggregateIndividualFields', + options: [ + { + name: 'Individual Fields', + value: 'aggregateIndividualFields', + }, + { + name: 'All Item Data (Into a Single List)', + value: 'aggregateAllItemData', + }, + ], + displayOptions: { + show: { + resource: ['itemList'], + operation: ['aggregateItems'], + }, }, }, - options: [ - { - displayName: '', - name: 'fieldToAggregate', - values: [ - { - displayName: 'Input Field Name', - name: 'fieldToAggregate', - type: 'string', - default: '', - description: 'The name of a field in the input items to aggregate together', - // eslint-disable-next-line n8n-nodes-base/node-param-placeholder-miscased-id - placeholder: 'e.g. id', - hint: ' Enter the field name as text', - requiresDataPath: 'single', - }, - { - displayName: 'Rename Field', - name: 'renameField', - type: 'boolean', - default: false, - description: 'Whether to give the field a different name in the output', - }, - { - displayName: 'Output Field Name', - name: 'outputFieldName', - displayOptions: { - show: { - renameField: [true], - }, - }, - type: 'string', - default: '', - description: - 'The name of the field to put the aggregated data in. Leave blank to use the input field name.', - requiresDataPath: 'single', - }, - ], + // Aggregate Individual Fields + { + displayName: 'Fields To Aggregate', + name: 'fieldsToAggregate', + type: 'fixedCollection', + typeOptions: { + multipleValues: true, }, - ], - }, - // Aggregate All Item Data - { - displayName: 'Put Output in Field', - name: 'destinationFieldName', - type: 'string', - displayOptions: { - show: { - resource: ['itemList'], - operation: ['aggregateItems'], - aggregate: ['aggregateAllItemData'], + placeholder: 'Add Field To Aggregate', + default: { fieldToAggregate: [{ fieldToAggregate: '', renameField: false }] }, + displayOptions: { + show: { + resource: ['itemList'], + operation: ['aggregateItems'], + aggregate: ['aggregateIndividualFields'], + }, }, + options: [ + { + displayName: '', + name: 'fieldToAggregate', + values: [ + { + displayName: 'Input Field Name', + name: 'fieldToAggregate', + type: 'string', + default: '', + description: 'The name of a field in the input items to aggregate together', + // eslint-disable-next-line n8n-nodes-base/node-param-placeholder-miscased-id + placeholder: 'e.g. id', + hint: ' Enter the field name as text', + requiresDataPath: 'single', + }, + { + displayName: 'Rename Field', + name: 'renameField', + type: 'boolean', + default: false, + description: 'Whether to give the field a different name in the output', + }, + { + displayName: 'Output Field Name', + name: 'outputFieldName', + displayOptions: { + show: { + renameField: [true], + }, + }, + type: 'string', + default: '', + description: + 'The name of the field to put the aggregated data in. Leave blank to use the input field name.', + requiresDataPath: 'single', + }, + ], + }, + ], }, - default: 'data', - description: 'The name of the output field to put the data in', - }, - { - displayName: 'Include', - name: 'include', - type: 'options', - default: 'allFields', - options: [ - { - name: 'All Fields', - value: 'allFields', - }, - { - name: 'Specified Fields', - value: 'specifiedFields', - }, - { - name: 'All Fields Except', - value: 'allFieldsExcept', - }, - ], - displayOptions: { - show: { - resource: ['itemList'], - operation: ['aggregateItems'], - aggregate: ['aggregateAllItemData'], + // Aggregate All Item Data + { + displayName: 'Put Output in Field', + name: 'destinationFieldName', + type: 'string', + displayOptions: { + show: { + resource: ['itemList'], + operation: ['aggregateItems'], + aggregate: ['aggregateAllItemData'], + }, }, + default: 'data', + description: 'The name of the output field to put the data in', }, - }, - { - displayName: 'Fields To Exclude', - name: 'fieldsToExclude', - type: 'fixedCollection', - typeOptions: { - multipleValues: true, - }, - placeholder: 'Add Field To Exclude', - default: {}, - options: [ - { - displayName: '', - name: 'fields', - values: [ - { - displayName: 'Field Name', - name: 'fieldName', - type: 'string', - default: '', - description: 'A field in the input to exclude from the object in output array', - // eslint-disable-next-line n8n-nodes-base/node-param-placeholder-miscased-id - placeholder: 'e.g. id', - hint: ' Enter the field name as text', - requiresDataPath: 'single', - }, - ], - }, - ], - displayOptions: { - show: { - resource: ['itemList'], - operation: ['aggregateItems'], - aggregate: ['aggregateAllItemData'], - include: ['allFieldsExcept'], + { + displayName: 'Include', + name: 'include', + type: 'options', + default: 'allFields', + options: [ + { + name: 'All Fields', + value: 'allFields', + }, + { + name: 'Specified Fields', + value: 'specifiedFields', + }, + { + name: 'All Fields Except', + value: 'allFieldsExcept', + }, + ], + displayOptions: { + show: { + resource: ['itemList'], + operation: ['aggregateItems'], + aggregate: ['aggregateAllItemData'], + }, }, }, - }, - { - displayName: 'Fields To Include', - name: 'fieldsToInclude', - type: 'fixedCollection', - typeOptions: { - multipleValues: true, - }, - placeholder: 'Add Field To Include', - default: {}, - options: [ - { - displayName: '', - name: 'fields', - values: [ - { - displayName: 'Field Name', - name: 'fieldName', - type: 'string', - default: '', - description: 'Specify fields that will be included in output array', - // eslint-disable-next-line n8n-nodes-base/node-param-placeholder-miscased-id - placeholder: 'e.g. id', - hint: ' Enter the field name as text', - requiresDataPath: 'single', - }, - ], + { + displayName: 'Fields To Exclude', + name: 'fieldsToExclude', + type: 'fixedCollection', + typeOptions: { + multipleValues: true, }, - ], - displayOptions: { - show: { - resource: ['itemList'], - operation: ['aggregateItems'], - aggregate: ['aggregateAllItemData'], - include: ['specifiedFields'], + placeholder: 'Add Field To Exclude', + default: {}, + options: [ + { + displayName: '', + name: 'fields', + values: [ + { + displayName: 'Field Name', + name: 'fieldName', + type: 'string', + default: '', + description: 'A field in the input to exclude from the object in output array', + // eslint-disable-next-line n8n-nodes-base/node-param-placeholder-miscased-id + placeholder: 'e.g. id', + hint: ' Enter the field name as text', + requiresDataPath: 'single', + }, + ], + }, + ], + displayOptions: { + show: { + resource: ['itemList'], + operation: ['aggregateItems'], + aggregate: ['aggregateAllItemData'], + include: ['allFieldsExcept'], + }, }, }, - }, - // Remove duplicates - Fields - { - displayName: 'Compare', - name: 'compare', - type: 'options', - options: [ - { - name: 'All Fields', - value: 'allFields', - }, - { - name: 'All Fields Except', - value: 'allFieldsExcept', + { + displayName: 'Fields To Include', + name: 'fieldsToInclude', + type: 'fixedCollection', + typeOptions: { + multipleValues: true, }, - { - name: 'Selected Fields', - value: 'selectedFields', - }, - ], - default: 'allFields', - description: 'The fields of the input items to compare to see if they are the same', - displayOptions: { - show: { - resource: ['itemList'], - operation: ['removeDuplicates'], + placeholder: 'Add Field To Include', + default: {}, + options: [ + { + displayName: '', + name: 'fields', + values: [ + { + displayName: 'Field Name', + name: 'fieldName', + type: 'string', + default: '', + description: 'Specify fields that will be included in output array', + // eslint-disable-next-line n8n-nodes-base/node-param-placeholder-miscased-id + placeholder: 'e.g. id', + hint: ' Enter the field name as text', + requiresDataPath: 'single', + }, + ], + }, + ], + displayOptions: { + show: { + resource: ['itemList'], + operation: ['aggregateItems'], + aggregate: ['aggregateAllItemData'], + include: ['specifiedFields'], + }, }, }, - }, - { - displayName: 'Fields To Exclude', - name: 'fieldsToExclude', - type: 'fixedCollection', - typeOptions: { - multipleValues: true, - }, - placeholder: 'Add Field To Exclude', - default: {}, - displayOptions: { - show: { - resource: ['itemList'], - operation: ['removeDuplicates'], - compare: ['allFieldsExcept'], + // Remove duplicates - Fields + { + displayName: 'Compare', + name: 'compare', + type: 'options', + options: [ + { + name: 'All Fields', + value: 'allFields', + }, + { + name: 'All Fields Except', + value: 'allFieldsExcept', + }, + { + name: 'Selected Fields', + value: 'selectedFields', + }, + ], + default: 'allFields', + description: 'The fields of the input items to compare to see if they are the same', + displayOptions: { + show: { + resource: ['itemList'], + operation: ['removeDuplicates'], + }, }, }, - options: [ - { - displayName: '', - name: 'fields', - values: [ - { - displayName: 'Field Name', - name: 'fieldName', - type: 'string', - default: '', - description: 'A field in the input to exclude from the comparison', - // eslint-disable-next-line n8n-nodes-base/node-param-placeholder-miscased-id - placeholder: 'e.g. id', - hint: ' Enter the field name as text', - requiresDataPath: 'single', - }, - ], + { + displayName: 'Fields To Exclude', + name: 'fieldsToExclude', + type: 'fixedCollection', + typeOptions: { + multipleValues: true, }, - ], - }, - { - displayName: 'Fields To Compare', - name: 'fieldsToCompare', - type: 'fixedCollection', - typeOptions: { - multipleValues: true, - }, - placeholder: 'Add Field To Compare', - default: {}, - displayOptions: { - show: { - resource: ['itemList'], - operation: ['removeDuplicates'], - compare: ['selectedFields'], + placeholder: 'Add Field To Exclude', + default: {}, + displayOptions: { + show: { + resource: ['itemList'], + operation: ['removeDuplicates'], + compare: ['allFieldsExcept'], + }, }, + options: [ + { + displayName: '', + name: 'fields', + values: [ + { + displayName: 'Field Name', + name: 'fieldName', + type: 'string', + default: '', + description: 'A field in the input to exclude from the comparison', + // eslint-disable-next-line n8n-nodes-base/node-param-placeholder-miscased-id + placeholder: 'e.g. id', + hint: ' Enter the field name as text', + requiresDataPath: 'single', + }, + ], + }, + ], }, - options: [ - { - displayName: '', - name: 'fields', - values: [ - { - displayName: 'Field Name', - name: 'fieldName', - type: 'string', - default: '', - description: 'A field in the input to add to the comparison', - // eslint-disable-next-line n8n-nodes-base/node-param-placeholder-miscased-id - placeholder: 'e.g. id', - hint: ' Enter the field name as text', - requiresDataPath: 'single', - }, - ], - }, - ], - }, - // Sort - Fields - { - displayName: 'Type', - name: 'type', - type: 'options', - options: [ - { - name: 'Simple', - value: 'simple', - }, - { - name: 'Random', - value: 'random', - }, - { - name: 'Code', - value: 'code', + { + displayName: 'Fields To Compare', + name: 'fieldsToCompare', + type: 'fixedCollection', + typeOptions: { + multipleValues: true, }, - ], - default: 'simple', - description: 'The fields of the input items to compare to see if they are the same', - displayOptions: { - show: { - resource: ['itemList'], - operation: ['sort'], + placeholder: 'Add Field To Compare', + default: {}, + displayOptions: { + show: { + resource: ['itemList'], + operation: ['removeDuplicates'], + compare: ['selectedFields'], + }, }, + options: [ + { + displayName: '', + name: 'fields', + values: [ + { + displayName: 'Field Name', + name: 'fieldName', + type: 'string', + default: '', + description: 'A field in the input to add to the comparison', + // eslint-disable-next-line n8n-nodes-base/node-param-placeholder-miscased-id + placeholder: 'e.g. id', + hint: ' Enter the field name as text', + requiresDataPath: 'single', + }, + ], + }, + ], }, - }, - { - displayName: 'Fields To Sort By', - name: 'sortFieldsUi', - type: 'fixedCollection', - typeOptions: { - multipleValues: true, + // Sort - Fields + { + displayName: 'Type', + name: 'type', + type: 'options', + options: [ + { + name: 'Simple', + value: 'simple', + }, + { + name: 'Random', + value: 'random', + }, + { + name: 'Code', + value: 'code', + }, + ], + default: 'simple', + description: 'The fields of the input items to compare to see if they are the same', + displayOptions: { + show: { + resource: ['itemList'], + operation: ['sort'], + }, + }, }, - placeholder: 'Add Field To Sort By', - options: [ - { - displayName: '', - name: 'sortField', - values: [ - { - displayName: 'Field Name', - name: 'fieldName', - type: 'string', - required: true, - default: '', - description: 'The field to sort by', - // eslint-disable-next-line n8n-nodes-base/node-param-placeholder-miscased-id - placeholder: 'e.g. id', - hint: ' Enter the field name as text', - requiresDataPath: 'single', - }, - { - displayName: 'Order', - name: 'order', - type: 'options', - options: [ - { - name: 'Ascending', - value: 'ascending', - }, - { - name: 'Descending', - value: 'descending', - }, - ], - default: 'ascending', - description: 'The order to sort by', - }, - ], + { + displayName: 'Fields To Sort By', + name: 'sortFieldsUi', + type: 'fixedCollection', + typeOptions: { + multipleValues: true, }, - ], - default: {}, - description: 'The fields of the input items to compare to see if they are the same', - displayOptions: { - show: { - resource: ['itemList'], - operation: ['sort'], - type: ['simple'], + placeholder: 'Add Field To Sort By', + options: [ + { + displayName: '', + name: 'sortField', + values: [ + { + displayName: 'Field Name', + name: 'fieldName', + type: 'string', + required: true, + default: '', + description: 'The field to sort by', + // eslint-disable-next-line n8n-nodes-base/node-param-placeholder-miscased-id + placeholder: 'e.g. id', + hint: ' Enter the field name as text', + requiresDataPath: 'single', + }, + { + displayName: 'Order', + name: 'order', + type: 'options', + options: [ + { + name: 'Ascending', + value: 'ascending', + }, + { + name: 'Descending', + value: 'descending', + }, + ], + default: 'ascending', + description: 'The order to sort by', + }, + ], + }, + ], + default: {}, + description: 'The fields of the input items to compare to see if they are the same', + displayOptions: { + show: { + resource: ['itemList'], + operation: ['sort'], + type: ['simple'], + }, }, }, - }, - { - displayName: 'Code', - name: 'code', - type: 'string', - typeOptions: { - alwaysOpenEditWindow: true, - editor: 'code', - rows: 10, - }, - default: `// The two items to compare are in the variables a and b + { + displayName: 'Code', + name: 'code', + type: 'string', + typeOptions: { + alwaysOpenEditWindow: true, + editor: 'code', + rows: 10, + }, + default: `// The two items to compare are in the variables a and b // Access the fields in a.json and b.json // Return -1 if a should go before b // Return 1 if b should go before a @@ -617,184 +616,185 @@ if (a.json[fieldName] > b.json[fieldName]) { return 1; } return 0;`, - description: 'Javascript code to determine the order of any two items', - displayOptions: { - show: { - resource: ['itemList'], - operation: ['sort'], - type: ['code'], - }, - }, - }, - // Limit - Fields - { - displayName: 'Max Items', - name: 'maxItems', - type: 'number', - typeOptions: { - minValue: 1, - }, - default: 1, - description: 'If there are more items than this number, some are removed', - displayOptions: { - show: { - resource: ['itemList'], - operation: ['limit'], + description: 'Javascript code to determine the order of any two items', + displayOptions: { + show: { + resource: ['itemList'], + operation: ['sort'], + type: ['code'], + }, }, }, - }, - { - displayName: 'Keep', - name: 'keep', - type: 'options', - options: [ - { - name: 'First Items', - value: 'firstItems', - }, - { - name: 'Last Items', - value: 'lastItems', + // Limit - Fields + { + displayName: 'Max Items', + name: 'maxItems', + type: 'number', + typeOptions: { + minValue: 1, }, - ], - default: 'firstItems', - description: 'When removing items, whether to keep the ones at the start or the ending', - displayOptions: { - show: { - resource: ['itemList'], - operation: ['limit'], + default: 1, + description: 'If there are more items than this number, some are removed', + displayOptions: { + show: { + resource: ['itemList'], + operation: ['limit'], + }, }, }, - }, - { - displayName: 'Options', - name: 'options', - type: 'collection', - placeholder: 'Add Field', - default: {}, - displayOptions: { - show: { - resource: ['itemList'], - operation: ['removeDuplicates'], - compare: ['allFieldsExcept', 'selectedFields'], + { + displayName: 'Keep', + name: 'keep', + type: 'options', + options: [ + { + name: 'First Items', + value: 'firstItems', + }, + { + name: 'Last Items', + value: 'lastItems', + }, + ], + default: 'firstItems', + description: 'When removing items, whether to keep the ones at the start or the ending', + displayOptions: { + show: { + resource: ['itemList'], + operation: ['limit'], + }, }, }, - options: [ - { - displayName: 'Remove Other Fields', - name: 'removeOtherFields', - type: 'boolean', - default: false, - description: - 'Whether to remove any fields that are not being compared. If disabled, will keep the values from the first of the duplicates.', - }, - { - displayName: 'Disable Dot Notation', - name: 'disableDotNotation', - type: 'boolean', - default: false, - description: - 'Whether to disallow referencing child fields using `parent.child` in the field name', - }, - ], - }, - { - displayName: 'Options', - name: 'options', - type: 'collection', - placeholder: 'Add Field', - default: {}, - displayOptions: { - show: { - resource: ['itemList'], - operation: ['sort'], - type: ['simple'], + { + displayName: 'Options', + name: 'options', + type: 'collection', + placeholder: 'Add Field', + default: {}, + displayOptions: { + show: { + resource: ['itemList'], + operation: ['removeDuplicates'], + compare: ['allFieldsExcept', 'selectedFields'], + }, }, + options: [ + { + displayName: 'Remove Other Fields', + name: 'removeOtherFields', + type: 'boolean', + default: false, + description: + 'Whether to remove any fields that are not being compared. If disabled, will keep the values from the first of the duplicates.', + }, + { + displayName: 'Disable Dot Notation', + name: 'disableDotNotation', + type: 'boolean', + default: false, + description: + 'Whether to disallow referencing child fields using `parent.child` in the field name', + }, + ], }, - options: [ - { - displayName: 'Disable Dot Notation', - name: 'disableDotNotation', - type: 'boolean', - default: false, - description: - 'Whether to disallow referencing child fields using `parent.child` in the field name', - }, - ], - }, - { - displayName: 'Options', - name: 'options', - type: 'collection', - placeholder: 'Add Field', - default: {}, - displayOptions: { - show: { - resource: ['itemList'], - operation: ['splitOutItems', 'aggregateItems'], - }, - hide: { - aggregate: ['aggregateAllItemData'], + { + displayName: 'Options', + name: 'options', + type: 'collection', + placeholder: 'Add Field', + default: {}, + displayOptions: { + show: { + resource: ['itemList'], + operation: ['sort'], + type: ['simple'], + }, }, + options: [ + { + displayName: 'Disable Dot Notation', + name: 'disableDotNotation', + type: 'boolean', + default: false, + description: + 'Whether to disallow referencing child fields using `parent.child` in the field name', + }, + ], }, - options: [ - { - displayName: 'Disable Dot Notation', - name: 'disableDotNotation', - type: 'boolean', - displayOptions: { - show: { - '/operation': ['splitOutItems', 'aggregateItems'], - }, + { + displayName: 'Options', + name: 'options', + type: 'collection', + placeholder: 'Add Field', + default: {}, + displayOptions: { + show: { + resource: ['itemList'], + operation: ['splitOutItems', 'aggregateItems'], + }, + hide: { + aggregate: ['aggregateAllItemData'], }, - default: false, - description: - 'Whether to disallow referencing child fields using `parent.child` in the field name', }, - { - displayName: 'Destination Field Name', - name: 'destinationFieldName', - type: 'string', - displayOptions: { - show: { - '/operation': ['splitOutItems'], + options: [ + { + displayName: 'Disable Dot Notation', + name: 'disableDotNotation', + type: 'boolean', + displayOptions: { + show: { + '/operation': ['splitOutItems', 'aggregateItems'], + }, }, + default: false, + description: + 'Whether to disallow referencing child fields using `parent.child` in the field name', }, - default: '', - description: 'The field in the output under which to put the split field contents', - }, - { - displayName: 'Merge Lists', - name: 'mergeLists', - type: 'boolean', - displayOptions: { - show: { - '/operation': ['aggregateItems'], + { + displayName: 'Destination Field Name', + name: 'destinationFieldName', + type: 'string', + displayOptions: { + show: { + '/operation': ['splitOutItems'], + }, }, + default: '', + description: 'The field in the output under which to put the split field contents', }, - default: false, - description: - 'Whether to merge the output into a single flat list (rather than a list of lists), if the field to aggregate is a list', - }, - { - displayName: 'Keep Missing And Null Values', - name: 'keepMissing', - type: 'boolean', - displayOptions: { - show: { - '/operation': ['aggregateItems'], + { + displayName: 'Merge Lists', + name: 'mergeLists', + type: 'boolean', + displayOptions: { + show: { + '/operation': ['aggregateItems'], + }, }, + default: false, + description: + 'Whether to merge the output into a single flat list (rather than a list of lists), if the field to aggregate is a list', }, - default: false, - description: - 'Whether to add a null entry to the aggregated list when there is a missing or null value', - }, - ], - }, - // Remove duplicates - Fields - ...summarize.description, - ], - }; + { + displayName: 'Keep Missing And Null Values', + name: 'keepMissing', + type: 'boolean', + displayOptions: { + show: { + '/operation': ['aggregateItems'], + }, + }, + default: false, + description: + 'Whether to add a null entry to the aggregated list when there is a missing or null value', + }, + ], + }, + // Remove duplicates - Fields + ...summarize.description, + ], + }; + } async execute(this: IExecuteFunctions): Promise { const items = this.getInputData(); diff --git a/packages/nodes-base/nodes/ItemLists/V1/summarize.operation.ts b/packages/nodes-base/nodes/ItemLists/V1/summarize.operation.ts new file mode 100644 index 0000000000000..0e0db54bcb7c0 --- /dev/null +++ b/packages/nodes-base/nodes/ItemLists/V1/summarize.operation.ts @@ -0,0 +1,582 @@ +import type { + GenericValue, + IDataObject, + IExecuteFunctions, + INodeExecutionData, + INodeProperties, +} from 'n8n-workflow'; +import { NodeOperationError } from 'n8n-workflow'; + +import get from 'lodash.get'; + +type AggregationType = + | 'append' + | 'average' + | 'concatenate' + | 'count' + | 'countUnique' + | 'max' + | 'min' + | 'sum'; + +type Aggregation = { + aggregation: AggregationType; + field: string; + includeEmpty?: boolean; + separateBy?: string; + customSeparator?: string; +}; + +type Aggregations = Aggregation[]; + +enum AggregationDisplayNames { + append = 'appended_', + average = 'average_', + concatenate = 'concatenated_', + count = 'count_', + countUnique = 'unique_count_', + max = 'max_', + min = 'min_', + sum = 'sum_', +} + +const NUMERICAL_AGGREGATIONS = ['average', 'max', 'min', 'sum']; + +type SummarizeOptions = { + disableDotNotation?: boolean; + outputFormat?: 'separateItems' | 'singleItem'; + skipEmptySplitFields?: boolean; +}; + +type ValueGetterFn = ( + item: IDataObject, + field: string, +) => IDataObject | IDataObject[] | GenericValue | GenericValue[]; + +export const description: INodeProperties[] = [ + { + displayName: 'Fields to Summarize', + name: 'fieldsToSummarize', + type: 'fixedCollection', + placeholder: 'Add Field', + default: { values: [{ aggregation: 'count', field: '' }] }, + typeOptions: { + multipleValues: true, + }, + options: [ + { + displayName: '', + name: 'values', + values: [ + { + displayName: 'Aggregation', + name: 'aggregation', + type: 'options', + options: [ + { + name: 'Append', + value: 'append', + }, + { + name: 'Average', + value: 'average', + }, + { + name: 'Concatenate', + value: 'concatenate', + }, + { + name: 'Count', + value: 'count', + }, + { + name: 'Count Unique', + value: 'countUnique', + }, + { + name: 'Max', + value: 'max', + }, + { + name: 'Min', + value: 'min', + }, + { + name: 'Sum', + value: 'sum', + }, + ], + default: 'count', + description: 'How to combine the values of the field you want to summarize', + }, + //field repeated to have different descriptions for different aggregations -------------------------------- + { + displayName: 'Field', + name: 'field', + type: 'string', + default: '', + description: 'The name of an input field that you want to summarize', + placeholder: 'e.g. cost', + hint: ' Enter the field name as text', + displayOptions: { + hide: { + aggregation: [...NUMERICAL_AGGREGATIONS, 'countUnique', 'count'], + }, + }, + requiresDataPath: 'single', + }, + { + displayName: 'Field', + name: 'field', + type: 'string', + default: '', + description: + 'The name of an input field that you want to summarize. The field should contain numerical values; null, undefined, empty strings would be ignored.', + placeholder: 'e.g. cost', + hint: ' Enter the field name as text', + displayOptions: { + show: { + aggregation: NUMERICAL_AGGREGATIONS, + }, + }, + requiresDataPath: 'single', + }, + { + displayName: 'Field', + name: 'field', + type: 'string', + default: '', + description: + 'The name of an input field that you want to summarize; null, undefined, empty strings would be ignored', + placeholder: 'e.g. cost', + hint: ' Enter the field name as text', + displayOptions: { + show: { + aggregation: ['countUnique', 'count'], + }, + }, + requiresDataPath: 'single', + }, + // ---------------------------------------------------------------------------------------------------------- + { + displayName: 'Include Empty Values', + name: 'includeEmpty', + type: 'boolean', + default: false, + displayOptions: { + show: { + aggregation: ['append', 'concatenate'], + }, + }, + }, + { + displayName: 'Separator', + name: 'separateBy', + type: 'options', + default: ',', + // eslint-disable-next-line n8n-nodes-base/node-param-options-type-unsorted-items + options: [ + { + name: 'Comma', + value: ',', + }, + { + name: 'Comma and Space', + value: ', ', + }, + { + name: 'New Line', + value: '\n', + }, + { + name: 'None', + value: '', + }, + { + name: 'Space', + value: ' ', + }, + { + name: 'Other', + value: 'other', + }, + ], + hint: 'What to insert between values', + displayOptions: { + show: { + aggregation: ['concatenate'], + }, + }, + }, + { + displayName: 'Custom Separator', + name: 'customSeparator', + type: 'string', + default: '', + displayOptions: { + show: { + aggregation: ['concatenate'], + separateBy: ['other'], + }, + }, + }, + ], + }, + ], + displayOptions: { + show: { + resource: ['itemList'], + operation: ['summarize'], + }, + }, + }, + // fieldsToSplitBy repeated to have different displayName for singleItem and separateItems ----------------------------- + { + displayName: 'Fields to Split By', + name: 'fieldsToSplitBy', + type: 'string', + placeholder: 'e.g. country, city', + default: '', + description: 'The name of the input fields that you want to split the summary by', + hint: 'Enter the name of the fields as text (separated by commas)', + displayOptions: { + show: { + resource: ['itemList'], + operation: ['summarize'], + }, + hide: { + '/options.outputFormat': ['singleItem'], + }, + }, + requiresDataPath: 'multiple', + }, + { + displayName: 'Fields to Group By', + name: 'fieldsToSplitBy', + type: 'string', + placeholder: 'e.g. country, city', + default: '', + description: 'The name of the input fields that you want to split the summary by', + hint: 'Enter the name of the fields as text (separated by commas)', + displayOptions: { + show: { + resource: ['itemList'], + operation: ['summarize'], + '/options.outputFormat': ['singleItem'], + }, + }, + requiresDataPath: 'multiple', + }, + // ---------------------------------------------------------------------------------------------------------- + { + displayName: 'Options', + name: 'options', + type: 'collection', + placeholder: 'Add Option', + default: {}, + displayOptions: { + show: { + resource: ['itemList'], + operation: ['summarize'], + }, + }, + options: [ + { + displayName: 'Disable Dot Notation', + name: 'disableDotNotation', + type: 'boolean', + default: false, + description: + 'Whether to disallow referencing child fields using `parent.child` in the field name', + }, + { + displayName: 'Output Format', + name: 'outputFormat', + type: 'options', + default: 'separateItems', + options: [ + { + name: 'Each Split in a Separate Item', + value: 'separateItems', + }, + { + name: 'All Splits in a Single Item', + value: 'singleItem', + }, + ], + }, + { + // eslint-disable-next-line n8n-nodes-base/node-param-display-name-miscased + displayName: 'Ignore items without valid fields to group by', + name: 'skipEmptySplitFields', + type: 'boolean', + default: false, + }, + ], + }, +]; + +function isEmpty(value: T) { + return value === undefined || value === null || value === ''; +} + +const fieldValueGetter = (disableDotNotation?: boolean) => { + if (disableDotNotation) { + return (item: IDataObject, field: string) => item[field]; + } else { + return (item: IDataObject, field: string) => get(item, field); + } +}; + +function checkIfFieldExists( + this: IExecuteFunctions, + items: IDataObject[], + aggregations: Aggregations, + getValue: ValueGetterFn, +) { + for (const aggregation of aggregations) { + if (aggregation.field === '') { + continue; + } + const exist = items.some((item) => getValue(item, aggregation.field) !== undefined); + if (!exist) { + throw new NodeOperationError( + this.getNode(), + `The field '${aggregation.field}' does not exist in any items`, + ); + } + } +} + +function aggregate(items: IDataObject[], entry: Aggregation, getValue: ValueGetterFn) { + const { aggregation, field } = entry; + let data = [...items]; + + if (NUMERICAL_AGGREGATIONS.includes(aggregation)) { + data = data.filter( + (item) => typeof getValue(item, field) === 'number' && !isEmpty(getValue(item, field)), + ); + } + + switch (aggregation) { + //combine operations + case 'append': + if (!entry.includeEmpty) { + data = data.filter((item) => !isEmpty(getValue(item, field))); + } + return data.map((item) => getValue(item, field)); + case 'concatenate': + const separateBy = entry.separateBy === 'other' ? entry.customSeparator : entry.separateBy; + if (!entry.includeEmpty) { + data = data.filter((item) => !isEmpty(getValue(item, field))); + } + return data + .map((item) => { + let value = getValue(item, field); + if (typeof value === 'object') { + value = JSON.stringify(value); + } + if (typeof value === 'undefined') { + value = 'undefined'; + } + + return value; + }) + .join(separateBy); + + //numerical operations + case 'average': + return ( + data.reduce((acc, item) => { + return acc + (getValue(item, field) as number); + }, 0) / data.length + ); + case 'sum': + return data.reduce((acc, item) => { + return acc + (getValue(item, field) as number); + }, 0); + case 'min': + return Math.min( + ...(data.map((item) => { + return getValue(item, field); + }) as number[]), + ); + case 'max': + return Math.max( + ...(data.map((item) => { + return getValue(item, field); + }) as number[]), + ); + + //count operations + case 'countUnique': + return new Set(data.map((item) => getValue(item, field)).filter((item) => !isEmpty(item))) + .size; + default: + //count by default + return data.filter((item) => !isEmpty(getValue(item, field))).length; + } +} + +function aggregateData( + data: IDataObject[], + fieldsToSummarize: Aggregations, + options: SummarizeOptions, + getValue: ValueGetterFn, +) { + const returnData = fieldsToSummarize.reduce((acc, aggregation) => { + acc[`${AggregationDisplayNames[aggregation.aggregation]}${aggregation.field}`] = aggregate( + data, + aggregation, + getValue, + ); + return acc; + }, {} as IDataObject); + if (options.outputFormat === 'singleItem') { + return returnData; + } else { + return { ...returnData, pairedItems: data.map((item) => item._itemIndex as number) }; + } +} + +function splitData( + splitKeys: string[], + data: IDataObject[], + fieldsToSummarize: Aggregations, + options: SummarizeOptions, + getValue: ValueGetterFn, +) { + if (!splitKeys || splitKeys.length === 0) { + return aggregateData(data, fieldsToSummarize, options, getValue); + } + + const [firstSplitKey, ...restSplitKeys] = splitKeys; + + const groupedData = data.reduce((acc, item) => { + let keyValuee = getValue(item, firstSplitKey) as string; + + if (typeof keyValuee === 'object') { + keyValuee = JSON.stringify(keyValuee); + } + + if (options.skipEmptySplitFields && typeof keyValuee !== 'number' && !keyValuee) { + return acc; + } + + if (acc[keyValuee] === undefined) { + acc[keyValuee] = [item]; + } else { + (acc[keyValuee] as IDataObject[]).push(item); + } + return acc; + }, {} as IDataObject); + + return Object.keys(groupedData).reduce((acc, key) => { + const value = groupedData[key] as IDataObject[]; + acc[key] = splitData(restSplitKeys, value, fieldsToSummarize, options, getValue); + return acc; + }, {} as IDataObject); +} + +function aggregationToArray( + aggregationResult: IDataObject, + fieldsToSplitBy: string[], + previousStage: IDataObject = {}, +) { + const returnData: IDataObject[] = []; + + const splitFieldName = fieldsToSplitBy[0]; + const isNext = fieldsToSplitBy[1]; + + if (isNext === undefined) { + for (const fieldName of Object.keys(aggregationResult)) { + returnData.push({ + ...previousStage, + [splitFieldName]: fieldName, + ...(aggregationResult[fieldName] as IDataObject), + }); + } + return returnData; + } else { + for (const key of Object.keys(aggregationResult)) { + returnData.push( + ...aggregationToArray(aggregationResult[key] as IDataObject, fieldsToSplitBy.slice(1), { + ...previousStage, + [splitFieldName]: key, + }), + ); + } + return returnData; + } +} + +export async function execute( + this: IExecuteFunctions, + items: INodeExecutionData[], +): Promise { + const newItems = items.map(({ json }, i) => ({ ...json, _itemIndex: i })); + + const options = this.getNodeParameter('options', 0, {}) as SummarizeOptions; + + const fieldsToSplitBy = (this.getNodeParameter('fieldsToSplitBy', 0, '') as string) + .split(',') + .map((field) => field.trim()) + .filter((field) => field); + + const fieldsToSummarize = this.getNodeParameter( + 'fieldsToSummarize.values', + 0, + [], + ) as Aggregations; + + if (fieldsToSummarize.filter((aggregation) => aggregation.field !== '').length === 0) { + throw new NodeOperationError( + this.getNode(), + "You need to add at least one aggregation to 'Fields to Summarize' with non empty 'Field'", + ); + } + + const getValue = fieldValueGetter(options.disableDotNotation); + + checkIfFieldExists.call(this, newItems, fieldsToSummarize, getValue); + + const aggregationResult = splitData( + fieldsToSplitBy, + newItems, + fieldsToSummarize, + options, + getValue, + ); + + if (options.outputFormat === 'singleItem') { + const executionData: INodeExecutionData = { + json: aggregationResult, + pairedItem: newItems.map((_v, index) => ({ + item: index, + })), + }; + return this.prepareOutputData([executionData]); + } else { + if (!fieldsToSplitBy.length) { + const { pairedItems, ...json } = aggregationResult; + const executionData: INodeExecutionData = { + json, + pairedItem: ((pairedItems as number[]) || []).map((index: number) => ({ + item: index, + })), + }; + return this.prepareOutputData([executionData]); + } + const returnData = aggregationToArray(aggregationResult, fieldsToSplitBy); + const executionData = returnData.map((item) => { + const { pairedItems, ...json } = item; + return { + json, + pairedItem: ((pairedItems as number[]) || []).map((index: number) => ({ + item: index, + })), + }; + }); + return this.prepareOutputData(executionData); + } +} diff --git a/packages/nodes-base/nodes/ItemLists/V2/ItemListsV2.node.ts b/packages/nodes-base/nodes/ItemLists/V2/ItemListsV2.node.ts new file mode 100644 index 0000000000000..04cf2f1808f50 --- /dev/null +++ b/packages/nodes-base/nodes/ItemLists/V2/ItemListsV2.node.ts @@ -0,0 +1,1428 @@ +import type { NodeVMOptions } from 'vm2'; +import { NodeVM } from 'vm2'; +import type { IExecuteFunctions } from 'n8n-core'; + +import type { + IDataObject, + INode, + INodeExecutionData, + INodeType, + INodeTypeBaseDescription, + INodeTypeDescription, +} from 'n8n-workflow'; +import { NodeOperationError } from 'n8n-workflow'; + +import get from 'lodash.get'; +import isEmpty from 'lodash.isempty'; +import isEqual from 'lodash.isequal'; +import isObject from 'lodash.isobject'; +import lt from 'lodash.lt'; +import merge from 'lodash.merge'; +import pick from 'lodash.pick'; +import reduce from 'lodash.reduce'; +import set from 'lodash.set'; +import unset from 'lodash.unset'; + +const compareItems = ( + obj: INodeExecutionData, + obj2: INodeExecutionData, + keys: string[], + disableDotNotation: boolean, + _node: INode, +) => { + let result = true; + for (const key of keys) { + if (!disableDotNotation) { + if (!isEqual(get(obj.json, key), get(obj2.json, key))) { + result = false; + break; + } + } else { + if (!isEqual(obj.json[key], obj2.json[key])) { + result = false; + break; + } + } + } + return result; +}; + +const flattenKeys = (obj: IDataObject, path: string[] = []): IDataObject => { + return !isObject(obj) + ? { [path.join('.')]: obj } + : reduce(obj, (cum, next, key) => merge(cum, flattenKeys(next as IDataObject, [...path, key])), {}); //prettier-ignore +}; + +const shuffleArray = (array: any[]) => { + for (let i = array.length - 1; i > 0; i--) { + const j = Math.floor(Math.random() * (i + 1)); + [array[i], array[j]] = [array[j], array[i]]; + } +}; + +import * as summarize from './summarize.operation'; + +export class ItemListsV2 implements INodeType { + description: INodeTypeDescription; + + constructor(baseDescription: INodeTypeBaseDescription) { + this.description = { + ...baseDescription, + version: 2, + defaults: { + name: 'Item Lists', + }, + inputs: ['main'], + outputs: ['main'], + credentials: [], + properties: [ + { + displayName: 'Resource', + name: 'resource', + type: 'hidden', + options: [ + { + name: 'Item List', + value: 'itemList', + }, + ], + default: 'itemList', + }, + { + displayName: 'Operation', + name: 'operation', + type: 'options', + noDataExpression: true, + options: [ + { + name: 'Concatenate Items', + value: 'aggregateItems', + description: 'Combine fields into a list in a single new item', + action: 'Combine fields into a list in a single new item', + }, + { + name: 'Limit', + value: 'limit', + description: 'Remove items if there are too many', + action: 'Remove items if there are too many', + }, + { + name: 'Remove Duplicates', + value: 'removeDuplicates', + description: 'Remove extra items that are similar', + action: 'Remove extra items that are similar', + }, + { + name: 'Sort', + value: 'sort', + description: 'Change the item order', + action: 'Change the item order', + }, + { + name: 'Split Out Items', + value: 'splitOutItems', + description: 'Turn a list inside item(s) into separate items', + action: 'Turn a list inside item(s) into separate items', + }, + { + name: 'Summarize', + value: 'summarize', + description: 'Aggregate items together (pivot table)', + action: 'Aggregate items together (pivot table)', + }, + ], + default: 'splitOutItems', + }, + // Split out items - Fields + { + displayName: 'Field To Split Out', + name: 'fieldToSplitOut', + type: 'string', + default: '', + required: true, + displayOptions: { + show: { + resource: ['itemList'], + operation: ['splitOutItems'], + }, + }, + description: 'The name of the input field to break out into separate items', + requiresDataPath: 'single', + }, + { + displayName: 'Include', + name: 'include', + type: 'options', + options: [ + { + name: 'No Other Fields', + value: 'noOtherFields', + }, + { + name: 'All Other Fields', + value: 'allOtherFields', + }, + { + name: 'Selected Other Fields', + value: 'selectedOtherFields', + }, + ], + default: 'noOtherFields', + description: 'Whether to copy any other fields into the new items', + displayOptions: { + show: { + resource: ['itemList'], + operation: ['splitOutItems'], + }, + }, + }, + { + displayName: 'Fields To Include', + name: 'fieldsToInclude', + type: 'fixedCollection', + typeOptions: { + multipleValues: true, + }, + placeholder: 'Add Field To Include', + default: {}, + displayOptions: { + show: { + resource: ['itemList'], + operation: ['splitOutItems'], + include: ['selectedOtherFields'], + }, + }, + options: [ + { + displayName: '', + name: 'fields', + values: [ + { + displayName: 'Field Name', + name: 'fieldName', + type: 'string', + default: '', + description: 'A field in the input items to aggregate together', + // eslint-disable-next-line n8n-nodes-base/node-param-placeholder-miscased-id + placeholder: 'e.g. id', + hint: ' Enter the field name as text', + requiresDataPath: 'single', + }, + ], + }, + ], + }, + // Aggregate Items + { + displayName: 'Aggregate', + name: 'aggregate', + type: 'options', + default: 'aggregateIndividualFields', + options: [ + { + name: 'Individual Fields', + value: 'aggregateIndividualFields', + }, + { + name: 'All Item Data (Into a Single List)', + value: 'aggregateAllItemData', + }, + ], + displayOptions: { + show: { + resource: ['itemList'], + operation: ['aggregateItems'], + }, + }, + }, + // Aggregate Individual Fields + { + displayName: 'Fields To Aggregate', + name: 'fieldsToAggregate', + type: 'fixedCollection', + typeOptions: { + multipleValues: true, + }, + placeholder: 'Add Field To Aggregate', + default: { fieldToAggregate: [{ fieldToAggregate: '', renameField: false }] }, + displayOptions: { + show: { + resource: ['itemList'], + operation: ['aggregateItems'], + aggregate: ['aggregateIndividualFields'], + }, + }, + options: [ + { + displayName: '', + name: 'fieldToAggregate', + values: [ + { + displayName: 'Input Field Name', + name: 'fieldToAggregate', + type: 'string', + default: '', + description: 'The name of a field in the input items to aggregate together', + // eslint-disable-next-line n8n-nodes-base/node-param-placeholder-miscased-id + placeholder: 'e.g. id', + hint: ' Enter the field name as text', + requiresDataPath: 'single', + }, + { + displayName: 'Rename Field', + name: 'renameField', + type: 'boolean', + default: false, + description: 'Whether to give the field a different name in the output', + }, + { + displayName: 'Output Field Name', + name: 'outputFieldName', + displayOptions: { + show: { + renameField: [true], + }, + }, + type: 'string', + default: '', + description: + 'The name of the field to put the aggregated data in. Leave blank to use the input field name.', + requiresDataPath: 'single', + }, + ], + }, + ], + }, + // Aggregate All Item Data + { + displayName: 'Put Output in Field', + name: 'destinationFieldName', + type: 'string', + displayOptions: { + show: { + resource: ['itemList'], + operation: ['aggregateItems'], + aggregate: ['aggregateAllItemData'], + }, + }, + default: 'data', + description: 'The name of the output field to put the data in', + }, + { + displayName: 'Include', + name: 'include', + type: 'options', + default: 'allFields', + options: [ + { + name: 'All Fields', + value: 'allFields', + }, + { + name: 'Specified Fields', + value: 'specifiedFields', + }, + { + name: 'All Fields Except', + value: 'allFieldsExcept', + }, + ], + displayOptions: { + show: { + resource: ['itemList'], + operation: ['aggregateItems'], + aggregate: ['aggregateAllItemData'], + }, + }, + }, + { + displayName: 'Fields To Exclude', + name: 'fieldsToExclude', + type: 'fixedCollection', + typeOptions: { + multipleValues: true, + }, + placeholder: 'Add Field To Exclude', + default: {}, + options: [ + { + displayName: '', + name: 'fields', + values: [ + { + displayName: 'Field Name', + name: 'fieldName', + type: 'string', + default: '', + description: 'A field in the input to exclude from the object in output array', + // eslint-disable-next-line n8n-nodes-base/node-param-placeholder-miscased-id + placeholder: 'e.g. id', + hint: ' Enter the field name as text', + requiresDataPath: 'single', + }, + ], + }, + ], + displayOptions: { + show: { + resource: ['itemList'], + operation: ['aggregateItems'], + aggregate: ['aggregateAllItemData'], + include: ['allFieldsExcept'], + }, + }, + }, + { + displayName: 'Fields To Include', + name: 'fieldsToInclude', + type: 'fixedCollection', + typeOptions: { + multipleValues: true, + }, + placeholder: 'Add Field To Include', + default: {}, + options: [ + { + displayName: '', + name: 'fields', + values: [ + { + displayName: 'Field Name', + name: 'fieldName', + type: 'string', + default: '', + description: 'Specify fields that will be included in output array', + // eslint-disable-next-line n8n-nodes-base/node-param-placeholder-miscased-id + placeholder: 'e.g. id', + hint: ' Enter the field name as text', + requiresDataPath: 'single', + }, + ], + }, + ], + displayOptions: { + show: { + resource: ['itemList'], + operation: ['aggregateItems'], + aggregate: ['aggregateAllItemData'], + include: ['specifiedFields'], + }, + }, + }, + // Remove duplicates - Fields + { + displayName: 'Compare', + name: 'compare', + type: 'options', + options: [ + { + name: 'All Fields', + value: 'allFields', + }, + { + name: 'All Fields Except', + value: 'allFieldsExcept', + }, + { + name: 'Selected Fields', + value: 'selectedFields', + }, + ], + default: 'allFields', + description: 'The fields of the input items to compare to see if they are the same', + displayOptions: { + show: { + resource: ['itemList'], + operation: ['removeDuplicates'], + }, + }, + }, + { + displayName: 'Fields To Exclude', + name: 'fieldsToExclude', + type: 'fixedCollection', + typeOptions: { + multipleValues: true, + }, + placeholder: 'Add Field To Exclude', + default: {}, + displayOptions: { + show: { + resource: ['itemList'], + operation: ['removeDuplicates'], + compare: ['allFieldsExcept'], + }, + }, + options: [ + { + displayName: '', + name: 'fields', + values: [ + { + displayName: 'Field Name', + name: 'fieldName', + type: 'string', + default: '', + description: 'A field in the input to exclude from the comparison', + // eslint-disable-next-line n8n-nodes-base/node-param-placeholder-miscased-id + placeholder: 'e.g. id', + hint: ' Enter the field name as text', + requiresDataPath: 'single', + }, + ], + }, + ], + }, + { + displayName: 'Fields To Compare', + name: 'fieldsToCompare', + type: 'fixedCollection', + typeOptions: { + multipleValues: true, + }, + placeholder: 'Add Field To Compare', + default: {}, + displayOptions: { + show: { + resource: ['itemList'], + operation: ['removeDuplicates'], + compare: ['selectedFields'], + }, + }, + options: [ + { + displayName: '', + name: 'fields', + values: [ + { + displayName: 'Field Name', + name: 'fieldName', + type: 'string', + default: '', + description: 'A field in the input to add to the comparison', + // eslint-disable-next-line n8n-nodes-base/node-param-placeholder-miscased-id + placeholder: 'e.g. id', + hint: ' Enter the field name as text', + requiresDataPath: 'single', + }, + ], + }, + ], + }, + // Sort - Fields + { + displayName: 'Type', + name: 'type', + type: 'options', + options: [ + { + name: 'Simple', + value: 'simple', + }, + { + name: 'Random', + value: 'random', + }, + { + name: 'Code', + value: 'code', + }, + ], + default: 'simple', + description: 'The fields of the input items to compare to see if they are the same', + displayOptions: { + show: { + resource: ['itemList'], + operation: ['sort'], + }, + }, + }, + { + displayName: 'Fields To Sort By', + name: 'sortFieldsUi', + type: 'fixedCollection', + typeOptions: { + multipleValues: true, + }, + placeholder: 'Add Field To Sort By', + options: [ + { + displayName: '', + name: 'sortField', + values: [ + { + displayName: 'Field Name', + name: 'fieldName', + type: 'string', + required: true, + default: '', + description: 'The field to sort by', + // eslint-disable-next-line n8n-nodes-base/node-param-placeholder-miscased-id + placeholder: 'e.g. id', + hint: ' Enter the field name as text', + requiresDataPath: 'single', + }, + { + displayName: 'Order', + name: 'order', + type: 'options', + options: [ + { + name: 'Ascending', + value: 'ascending', + }, + { + name: 'Descending', + value: 'descending', + }, + ], + default: 'ascending', + description: 'The order to sort by', + }, + ], + }, + ], + default: {}, + description: 'The fields of the input items to compare to see if they are the same', + displayOptions: { + show: { + resource: ['itemList'], + operation: ['sort'], + type: ['simple'], + }, + }, + }, + { + displayName: 'Code', + name: 'code', + type: 'string', + typeOptions: { + alwaysOpenEditWindow: true, + editor: 'code', + rows: 10, + }, + default: `// The two items to compare are in the variables a and b +// Access the fields in a.json and b.json +// Return -1 if a should go before b +// Return 1 if b should go before a +// Return 0 if there's no difference + +fieldName = 'myField'; + +if (a.json[fieldName] < b.json[fieldName]) { + return -1; +} +if (a.json[fieldName] > b.json[fieldName]) { + return 1; +} +return 0;`, + description: 'Javascript code to determine the order of any two items', + displayOptions: { + show: { + resource: ['itemList'], + operation: ['sort'], + type: ['code'], + }, + }, + }, + // Limit - Fields + { + displayName: 'Max Items', + name: 'maxItems', + type: 'number', + typeOptions: { + minValue: 1, + }, + default: 1, + description: 'If there are more items than this number, some are removed', + displayOptions: { + show: { + resource: ['itemList'], + operation: ['limit'], + }, + }, + }, + { + displayName: 'Keep', + name: 'keep', + type: 'options', + options: [ + { + name: 'First Items', + value: 'firstItems', + }, + { + name: 'Last Items', + value: 'lastItems', + }, + ], + default: 'firstItems', + description: 'When removing items, whether to keep the ones at the start or the ending', + displayOptions: { + show: { + resource: ['itemList'], + operation: ['limit'], + }, + }, + }, + { + displayName: 'Options', + name: 'options', + type: 'collection', + placeholder: 'Add Field', + default: {}, + displayOptions: { + show: { + resource: ['itemList'], + operation: ['removeDuplicates'], + compare: ['allFieldsExcept', 'selectedFields'], + }, + }, + options: [ + { + displayName: 'Remove Other Fields', + name: 'removeOtherFields', + type: 'boolean', + default: false, + description: + 'Whether to remove any fields that are not being compared. If disabled, will keep the values from the first of the duplicates.', + }, + { + displayName: 'Disable Dot Notation', + name: 'disableDotNotation', + type: 'boolean', + default: false, + description: + 'Whether to disallow referencing child fields using `parent.child` in the field name', + }, + ], + }, + { + displayName: 'Options', + name: 'options', + type: 'collection', + placeholder: 'Add Field', + default: {}, + displayOptions: { + show: { + resource: ['itemList'], + operation: ['sort'], + type: ['simple'], + }, + }, + options: [ + { + displayName: 'Disable Dot Notation', + name: 'disableDotNotation', + type: 'boolean', + default: false, + description: + 'Whether to disallow referencing child fields using `parent.child` in the field name', + }, + ], + }, + { + displayName: 'Options', + name: 'options', + type: 'collection', + placeholder: 'Add Field', + default: {}, + displayOptions: { + show: { + resource: ['itemList'], + operation: ['splitOutItems', 'aggregateItems'], + }, + hide: { + aggregate: ['aggregateAllItemData'], + }, + }, + options: [ + { + displayName: 'Disable Dot Notation', + name: 'disableDotNotation', + type: 'boolean', + displayOptions: { + show: { + '/operation': ['splitOutItems', 'aggregateItems'], + }, + }, + default: false, + description: + 'Whether to disallow referencing child fields using `parent.child` in the field name', + }, + { + displayName: 'Destination Field Name', + name: 'destinationFieldName', + type: 'string', + displayOptions: { + show: { + '/operation': ['splitOutItems'], + }, + }, + default: '', + description: 'The field in the output under which to put the split field contents', + }, + { + displayName: 'Merge Lists', + name: 'mergeLists', + type: 'boolean', + displayOptions: { + show: { + '/operation': ['aggregateItems'], + }, + }, + default: false, + description: + 'Whether to merge the output into a single flat list (rather than a list of lists), if the field to aggregate is a list', + }, + { + displayName: 'Keep Missing And Null Values', + name: 'keepMissing', + type: 'boolean', + displayOptions: { + show: { + '/operation': ['aggregateItems'], + }, + }, + default: false, + description: + 'Whether to add a null entry to the aggregated list when there is a missing or null value', + }, + ], + }, + // Remove duplicates - Fields + ...summarize.description, + ], + }; + } + + async execute(this: IExecuteFunctions): Promise { + const items = this.getInputData(); + const length = items.length; + const returnData: INodeExecutionData[] = []; + const resource = this.getNodeParameter('resource', 0); + const operation = this.getNodeParameter('operation', 0); + if (resource === 'itemList') { + if (operation === 'splitOutItems') { + for (let i = 0; i < length; i++) { + const fieldToSplitOut = this.getNodeParameter('fieldToSplitOut', i) as string; + const disableDotNotation = this.getNodeParameter( + 'options.disableDotNotation', + 0, + false, + ) as boolean; + const destinationFieldName = this.getNodeParameter( + 'options.destinationFieldName', + i, + '', + ) as string; + const include = this.getNodeParameter('include', i) as string; + + let arrayToSplit; + if (!disableDotNotation) { + arrayToSplit = get(items[i].json, fieldToSplitOut); + } else { + arrayToSplit = items[i].json[fieldToSplitOut]; + } + + if (arrayToSplit === undefined) { + if (fieldToSplitOut.includes('.') && disableDotNotation) { + throw new NodeOperationError( + this.getNode(), + `Couldn't find the field '${fieldToSplitOut}' in the input data`, + { + description: + "If you're trying to use a nested field, make sure you turn off 'disable dot notation' in the node options", + }, + ); + } else { + throw new NodeOperationError( + this.getNode(), + `Couldn't find the field '${fieldToSplitOut}' in the input data`, + { itemIndex: i }, + ); + } + } + + if (!Array.isArray(arrayToSplit)) { + throw new NodeOperationError( + this.getNode(), + `The provided field '${fieldToSplitOut}' is not an array`, + { itemIndex: i }, + ); + } else { + for (const element of arrayToSplit) { + let newItem = {}; + + if (include === 'selectedOtherFields') { + const fieldsToInclude = ( + this.getNodeParameter('fieldsToInclude.fields', i, []) as [{ fieldName: string }] + ).map((field) => field.fieldName); + + if (!fieldsToInclude.length) { + throw new NodeOperationError(this.getNode(), 'No fields specified', { + description: 'Please add a field to include', + }); + } + + newItem = { + ...fieldsToInclude.reduce((prev, field) => { + if (field === fieldToSplitOut) { + return prev; + } + let value; + if (!disableDotNotation) { + value = get(items[i].json, field); + } else { + value = items[i].json[field]; + } + prev = { ...prev, [field]: value }; + return prev; + }, {}), + }; + } else if (include === 'allOtherFields') { + const keys = Object.keys(items[i].json); + + newItem = { + ...keys.reduce((prev, field) => { + let value; + if (!disableDotNotation) { + value = get(items[i].json, field); + } else { + value = items[i].json[field]; + } + prev = { ...prev, [field]: value }; + return prev; + }, {}), + }; + + unset(newItem, fieldToSplitOut); + } + + if ( + typeof element === 'object' && + include === 'noOtherFields' && + destinationFieldName === '' + ) { + newItem = { ...newItem, ...element }; + } else { + newItem = { + ...newItem, + [destinationFieldName || fieldToSplitOut]: element, + }; + } + + returnData.push({ + json: newItem, + pairedItem: { + item: i, + }, + }); + } + } + } + + return this.prepareOutputData(returnData); + } else if (operation === 'aggregateItems') { + const aggregate = this.getNodeParameter('aggregate', 0, '') as string; + + if (aggregate === 'aggregateIndividualFields') { + const disableDotNotation = this.getNodeParameter( + 'options.disableDotNotation', + 0, + false, + ) as boolean; + const mergeLists = this.getNodeParameter('options.mergeLists', 0, false) as boolean; + const fieldsToAggregate = this.getNodeParameter( + 'fieldsToAggregate.fieldToAggregate', + 0, + [], + ) as [{ fieldToAggregate: string; renameField: boolean; outputFieldName: string }]; + const keepMissing = this.getNodeParameter('options.keepMissing', 0, false) as boolean; + + if (!fieldsToAggregate.length) { + throw new NodeOperationError(this.getNode(), 'No fields specified', { + description: 'Please add a field to aggregate', + }); + } + for (const { fieldToAggregate } of fieldsToAggregate) { + let found = false; + for (const item of items) { + if (fieldToAggregate === '') { + throw new NodeOperationError(this.getNode(), 'Field to aggregate is blank', { + description: 'Please add a field to aggregate', + }); + } + if (!disableDotNotation) { + if (get(item.json, fieldToAggregate) !== undefined) { + found = true; + } + } else if (item.json.hasOwnProperty(fieldToAggregate)) { + found = true; + } + } + if (!found && disableDotNotation && fieldToAggregate.includes('.')) { + throw new NodeOperationError( + this.getNode(), + `Couldn't find the field '${fieldToAggregate}' in the input data`, + { + description: + "If you're trying to use a nested field, make sure you turn off 'disable dot notation' in the node options", + }, + ); + } else if (!found && !keepMissing) { + throw new NodeOperationError( + this.getNode(), + `Couldn't find the field '${fieldToAggregate}' in the input data`, + ); + } + } + + const newItem: INodeExecutionData = { + json: {}, + pairedItem: Array.from({ length }, (_, i) => i).map((index) => { + return { + item: index, + }; + }), + }; + + const values: { [key: string]: any } = {}; + const outputFields: string[] = []; + + for (const { fieldToAggregate, outputFieldName, renameField } of fieldsToAggregate) { + const field = renameField ? outputFieldName : fieldToAggregate; + + if (outputFields.includes(field)) { + throw new NodeOperationError( + this.getNode(), + `The '${field}' output field is used more than once`, + { description: 'Please make sure each output field name is unique' }, + ); + } else { + outputFields.push(field); + } + + const getFieldToAggregate = () => + !disableDotNotation && fieldToAggregate.includes('.') + ? fieldToAggregate.split('.').pop() + : fieldToAggregate; + + const _outputFieldName = outputFieldName + ? outputFieldName + : (getFieldToAggregate() as string); + + if (fieldToAggregate !== '') { + values[_outputFieldName] = []; + for (let i = 0; i < length; i++) { + if (!disableDotNotation) { + let value = get(items[i].json, fieldToAggregate); + + if (!keepMissing) { + if (Array.isArray(value)) { + value = value.filter((entry) => entry !== null); + } else if (value === null || value === undefined) { + continue; + } + } + + if (Array.isArray(value) && mergeLists) { + values[_outputFieldName].push(...value); + } else { + values[_outputFieldName].push(value); + } + } else { + let value = items[i].json[fieldToAggregate]; + + if (!keepMissing) { + if (Array.isArray(value)) { + value = value.filter((entry) => entry !== null); + } else if (value === null || value === undefined) { + continue; + } + } + + if (Array.isArray(value) && mergeLists) { + values[_outputFieldName].push(...value); + } else { + values[_outputFieldName].push(value); + } + } + } + } + } + + for (const key of Object.keys(values)) { + if (!disableDotNotation) { + set(newItem.json, key, values[key]); + } else { + newItem.json[key] = values[key]; + } + } + + returnData.push(newItem); + + return this.prepareOutputData(returnData); + } else { + let newItems: IDataObject[] = items.map((item) => item.json); + const destinationFieldName = this.getNodeParameter('destinationFieldName', 0) as string; + const fieldsToExclude = ( + this.getNodeParameter('fieldsToExclude.fields', 0, []) as IDataObject[] + ).map((entry) => entry.fieldName); + const fieldsToInclude = ( + this.getNodeParameter('fieldsToInclude.fields', 0, []) as IDataObject[] + ).map((entry) => entry.fieldName); + + if (fieldsToExclude.length || fieldsToInclude.length) { + newItems = newItems.reduce((acc, item) => { + const newItem: IDataObject = {}; + let outputFields = Object.keys(item); + + if (fieldsToExclude.length) { + outputFields = outputFields.filter((key) => !fieldsToExclude.includes(key)); + } + if (fieldsToInclude.length) { + outputFields = outputFields.filter((key) => + fieldsToInclude.length ? fieldsToInclude.includes(key) : true, + ); + } + + outputFields.forEach((key) => { + newItem[key] = item[key]; + }); + + if (isEmpty(newItem)) { + return acc; + } + return acc.concat([newItem]); + }, [] as IDataObject[]); + } + + const output: INodeExecutionData = { json: { [destinationFieldName]: newItems } }; + + return this.prepareOutputData([output]); + } + } else if (operation === 'removeDuplicates') { + const compare = this.getNodeParameter('compare', 0) as string; + const disableDotNotation = this.getNodeParameter( + 'options.disableDotNotation', + 0, + false, + ) as boolean; + const removeOtherFields = this.getNodeParameter( + 'options.removeOtherFields', + 0, + false, + ) as boolean; + + let keys = disableDotNotation + ? Object.keys(items[0].json) + : Object.keys(flattenKeys(items[0].json)); + + for (const item of items) { + for (const key of disableDotNotation + ? Object.keys(item.json) + : Object.keys(flattenKeys(item.json))) { + if (!keys.includes(key)) { + keys.push(key); + } + } + } + + if (compare === 'allFieldsExcept') { + const fieldsToExclude = ( + this.getNodeParameter('fieldsToExclude.fields', 0, []) as [{ fieldName: string }] + ).map((field) => field.fieldName); + if (!fieldsToExclude.length) { + throw new NodeOperationError( + this.getNode(), + 'No fields specified. Please add a field to exclude from comparison', + ); + } + if (!disableDotNotation) { + keys = Object.keys(flattenKeys(items[0].json)); + } + keys = keys.filter((key) => !fieldsToExclude.includes(key)); + } + if (compare === 'selectedFields') { + const fieldsToCompare = ( + this.getNodeParameter('fieldsToCompare.fields', 0, []) as [{ fieldName: string }] + ).map((field) => field.fieldName); + if (!fieldsToCompare.length) { + throw new NodeOperationError( + this.getNode(), + 'No fields specified. Please add a field to compare on', + ); + } + if (!disableDotNotation) { + keys = Object.keys(flattenKeys(items[0].json)); + } + keys = fieldsToCompare.map((key) => key.trim()); + } + + // This solution is O(nlogn) + // add original index to the items + const newItems = items.map( + (item, index) => + ({ + json: { ...item.json, __INDEX: index }, + pairedItem: { item: index }, + } as INodeExecutionData), + ); + //sort items using the compare keys + newItems.sort((a, b) => { + let result = 0; + + for (const key of keys) { + let equal; + if (!disableDotNotation) { + equal = isEqual(get(a.json, key), get(b.json, key)); + } else { + equal = isEqual(a.json[key], b.json[key]); + } + if (!equal) { + let lessThan; + if (!disableDotNotation) { + lessThan = lt(get(a.json, key), get(b.json, key)); + } else { + lessThan = lt(a.json[key], b.json[key]); + } + result = lessThan ? -1 : 1; + break; + } + } + return result; + }); + + for (const key of keys) { + let type: any = undefined; + for (const item of newItems) { + if (key === '') { + throw new NodeOperationError(this.getNode(), 'Name of field to compare is blank'); + } + const value = !disableDotNotation ? get(item.json, key) : item.json[key]; + if (value === undefined && disableDotNotation && key.includes('.')) { + throw new NodeOperationError( + this.getNode(), + `'${key}' field is missing from some input items`, + { + description: + "If you're trying to use a nested field, make sure you turn off 'disable dot notation' in the node options", + }, + ); + } else if (value === undefined) { + throw new NodeOperationError( + this.getNode(), + `'${key}' field is missing from some input items`, + ); + } + if (type !== undefined && value !== undefined && type !== typeof value) { + throw new NodeOperationError(this.getNode(), `'${key}' isn't always the same type`, { + description: 'The type of this field varies between items', + }); + } else { + type = typeof value; + } + } + } + + // collect the original indexes of items to be removed + const removedIndexes: number[] = []; + let temp = newItems[0]; + for (let index = 1; index < newItems.length; index++) { + if (compareItems(newItems[index], temp, keys, disableDotNotation, this.getNode())) { + removedIndexes.push(newItems[index].json.__INDEX as unknown as number); + } else { + temp = newItems[index]; + } + } + + let data = items.filter((_, index) => !removedIndexes.includes(index)); + + if (removeOtherFields) { + data = data.map((item, index) => ({ + json: pick(item.json, ...keys), + pairedItem: { item: index }, + })); + } + + // return the filtered items + return this.prepareOutputData(data); + } else if (operation === 'sort') { + let newItems = [...items]; + const type = this.getNodeParameter('type', 0) as string; + const disableDotNotation = this.getNodeParameter( + 'options.disableDotNotation', + 0, + false, + ) as boolean; + + if (type === 'random') { + shuffleArray(newItems); + return this.prepareOutputData(newItems); + } + + if (type === 'simple') { + const sortFieldsUi = this.getNodeParameter('sortFieldsUi', 0) as IDataObject; + const sortFields = sortFieldsUi.sortField as Array<{ + fieldName: string; + order: 'ascending' | 'descending'; + }>; + + if (!sortFields?.length) { + throw new NodeOperationError( + this.getNode(), + 'No sorting specified. Please add a field to sort by', + ); + } + + for (const { fieldName } of sortFields) { + let found = false; + for (const item of items) { + if (!disableDotNotation) { + if (get(item.json, fieldName) !== undefined) { + found = true; + } + } else if (item.json.hasOwnProperty(fieldName)) { + found = true; + } + } + if (!found && disableDotNotation && fieldName.includes('.')) { + throw new NodeOperationError( + this.getNode(), + `Couldn't find the field '${fieldName}' in the input data`, + { + description: + "If you're trying to use a nested field, make sure you turn off 'disable dot notation' in the node options", + }, + ); + } else if (!found) { + throw new NodeOperationError( + this.getNode(), + `Couldn't find the field '${fieldName}' in the input data`, + ); + } + } + + const sortFieldsWithDirection = sortFields.map((field) => ({ + name: field.fieldName, + dir: field.order === 'ascending' ? 1 : -1, + })); + + newItems.sort((a, b) => { + let result = 0; + for (const field of sortFieldsWithDirection) { + let equal; + if (!disableDotNotation) { + const _a = + typeof get(a.json, field.name) === 'string' + ? (get(a.json, field.name) as string).toLowerCase() + : get(a.json, field.name); + const _b = + typeof get(b.json, field.name) === 'string' + ? (get(b.json, field.name) as string).toLowerCase() + : get(b.json, field.name); + equal = isEqual(_a, _b); + } else { + const _a = + typeof a.json[field.name] === 'string' + ? (a.json[field.name] as string).toLowerCase() + : a.json[field.name]; + const _b = + typeof b.json[field.name] === 'string' + ? (b.json[field.name] as string).toLowerCase() + : b.json[field.name]; + equal = isEqual(_a, _b); + } + + if (!equal) { + let lessThan; + if (!disableDotNotation) { + const _a = + typeof get(a.json, field.name) === 'string' + ? (get(a.json, field.name) as string).toLowerCase() + : get(a.json, field.name); + const _b = + typeof get(b.json, field.name) === 'string' + ? (get(b.json, field.name) as string).toLowerCase() + : get(b.json, field.name); + lessThan = lt(_a, _b); + } else { + const _a = + typeof a.json[field.name] === 'string' + ? (a.json[field.name] as string).toLowerCase() + : a.json[field.name]; + const _b = + typeof b.json[field.name] === 'string' + ? (b.json[field.name] as string).toLowerCase() + : b.json[field.name]; + lessThan = lt(_a, _b); + } + if (lessThan) { + result = -1 * field.dir; + } else { + result = 1 * field.dir; + } + break; + } + } + return result; + }); + } else { + const code = this.getNodeParameter('code', 0) as string; + const regexCheck = /\breturn\b/g.exec(code); + + if (regexCheck?.length) { + const sandbox = { + newItems, + }; + const mode = this.getMode(); + const options = { + console: mode === 'manual' ? 'redirect' : 'inherit', + sandbox, + }; + const vm = new NodeVM(options as unknown as NodeVMOptions); + + newItems = await vm.run( + ` + module.exports = async function() { + newItems.sort( (a,b) => { + ${code} + }) + return newItems; + }()`, + __dirname, + ); + } else { + throw new NodeOperationError( + this.getNode(), + "Sort code doesn't return. Please add a 'return' statement to your code", + ); + } + } + return this.prepareOutputData(newItems); + } else if (operation === 'limit') { + let newItems = items; + const maxItems = this.getNodeParameter('maxItems', 0) as number; + const keep = this.getNodeParameter('keep', 0) as string; + + if (maxItems > items.length) { + return this.prepareOutputData(newItems); + } + + if (keep === 'firstItems') { + newItems = items.slice(0, maxItems); + } else { + newItems = items.slice(items.length - maxItems, items.length); + } + return this.prepareOutputData(newItems); + } else if (operation === 'summarize') { + return summarize.execute.call(this, items); + } else { + throw new NodeOperationError(this.getNode(), `Operation '${operation}' is not recognized`); + } + } else { + throw new NodeOperationError(this.getNode(), `Resource '${resource}' is not recognized`); + } + } +} diff --git a/packages/nodes-base/nodes/ItemLists/summarize.operation.ts b/packages/nodes-base/nodes/ItemLists/V2/summarize.operation.ts similarity index 100% rename from packages/nodes-base/nodes/ItemLists/summarize.operation.ts rename to packages/nodes-base/nodes/ItemLists/V2/summarize.operation.ts From 23bc6b9df41d9e0a7f783beca14e67dc82bb4477 Mon Sep 17 00:00:00 2001 From: agobrech Date: Tue, 28 Feb 2023 15:34:37 +0100 Subject: [PATCH 6/7] Fix naming --- .../nodes/ItemLists/{ItemList.node.ts => ItemLists.node.ts} | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) rename packages/nodes-base/nodes/ItemLists/{ItemList.node.ts => ItemLists.node.ts} (86%) diff --git a/packages/nodes-base/nodes/ItemLists/ItemList.node.ts b/packages/nodes-base/nodes/ItemLists/ItemLists.node.ts similarity index 86% rename from packages/nodes-base/nodes/ItemLists/ItemList.node.ts rename to packages/nodes-base/nodes/ItemLists/ItemLists.node.ts index 542186e71b956..c76d73bf2fa2c 100644 --- a/packages/nodes-base/nodes/ItemLists/ItemList.node.ts +++ b/packages/nodes-base/nodes/ItemLists/ItemLists.node.ts @@ -1,3 +1,4 @@ +// eslint-disable-next-line n8n-nodes-base/node-dirname-against-convention import type { INodeTypeBaseDescription, IVersionedNodeType } from 'n8n-workflow'; import { VersionedNodeType } from 'n8n-workflow'; @@ -5,7 +6,7 @@ import { ItemListsV1 } from './V1/ItemListsV1.node'; import { ItemListsV2 } from './V2/ItemListsV2.node'; -export class ItemList extends VersionedNodeType { +export class ItemLists extends VersionedNodeType { constructor() { const baseDescription: INodeTypeBaseDescription = { displayName: 'Item Lists', From d9e25fb920132d6616188103fcbc30b0be0996a3 Mon Sep 17 00:00:00 2001 From: agobrech Date: Tue, 28 Feb 2023 15:39:24 +0100 Subject: [PATCH 7/7] Remove comment --- packages/nodes-base/nodes/ItemLists/ItemLists.node.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/nodes-base/nodes/ItemLists/ItemLists.node.ts b/packages/nodes-base/nodes/ItemLists/ItemLists.node.ts index c76d73bf2fa2c..25a640528043e 100644 --- a/packages/nodes-base/nodes/ItemLists/ItemLists.node.ts +++ b/packages/nodes-base/nodes/ItemLists/ItemLists.node.ts @@ -1,4 +1,3 @@ -// eslint-disable-next-line n8n-nodes-base/node-dirname-against-convention import type { INodeTypeBaseDescription, IVersionedNodeType } from 'n8n-workflow'; import { VersionedNodeType } from 'n8n-workflow';