From 85ce52e99da9f36c111232ce928566fca6dfd369 Mon Sep 17 00:00:00 2001 From: Elias Meire Date: Thu, 16 Nov 2023 16:49:59 +0100 Subject: [PATCH 01/17] Review node descriptions --- packages/nodes-base/nodes/DateTime/V2/DateTimeV2.node.ts | 1 + packages/nodes-base/nodes/Filter/Filter.node.ts | 2 +- packages/nodes-base/nodes/Merge/v2/MergeV2.node.ts | 2 +- packages/nodes-base/nodes/Set/v2/SetV2.node.ts | 2 +- 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/nodes-base/nodes/DateTime/V2/DateTimeV2.node.ts b/packages/nodes-base/nodes/DateTime/V2/DateTimeV2.node.ts index ae3f11a526b1c..4ee0f00e11e5a 100644 --- a/packages/nodes-base/nodes/DateTime/V2/DateTimeV2.node.ts +++ b/packages/nodes-base/nodes/DateTime/V2/DateTimeV2.node.ts @@ -31,6 +31,7 @@ export class DateTimeV2 implements INodeType { }, inputs: ['main'], outputs: ['main'], + description: 'Manipulate date and time values', properties: [ { displayName: 'Operation', diff --git a/packages/nodes-base/nodes/Filter/Filter.node.ts b/packages/nodes-base/nodes/Filter/Filter.node.ts index d5a3e84d27eba..76913e891f257 100644 --- a/packages/nodes-base/nodes/Filter/Filter.node.ts +++ b/packages/nodes-base/nodes/Filter/Filter.node.ts @@ -16,7 +16,7 @@ export class Filter implements INodeType { icon: 'fa:filter', group: ['transform'], version: 1, - description: 'Filter out incoming items based on given conditions', + description: 'Remove items matching a condition', defaults: { name: 'Filter', color: '#229eff', diff --git a/packages/nodes-base/nodes/Merge/v2/MergeV2.node.ts b/packages/nodes-base/nodes/Merge/v2/MergeV2.node.ts index 502422c4fb573..c246339c0ddf9 100644 --- a/packages/nodes-base/nodes/Merge/v2/MergeV2.node.ts +++ b/packages/nodes-base/nodes/Merge/v2/MergeV2.node.ts @@ -37,7 +37,7 @@ const versionDescription: INodeTypeDescription = { group: ['transform'], version: [2, 2.1, 2.2], subtitle: '={{$parameter["mode"]}}', - description: 'Merges data of multiple streams once data from both is available', + description: 'Merge data of two inputs once data from both is available', defaults: { name: 'Merge', color: '#00bbcc', diff --git a/packages/nodes-base/nodes/Set/v2/SetV2.node.ts b/packages/nodes-base/nodes/Set/v2/SetV2.node.ts index 62a860140166d..78135990fa6d3 100644 --- a/packages/nodes-base/nodes/Set/v2/SetV2.node.ts +++ b/packages/nodes-base/nodes/Set/v2/SetV2.node.ts @@ -22,7 +22,7 @@ const versionDescription: INodeTypeDescription = { icon: 'fa:pen', group: ['input'], version: [3, 3.1, 3.2], - description: 'Change the structure of your items', + description: 'Change fields of items, or create new fields', subtitle: '={{$parameter["mode"]}}', defaults: { name: 'Edit Fields', From 225698559103e205523a1447a13fc12c698b3431 Mon Sep 17 00:00:00 2001 From: Elias Meire Date: Fri, 17 Nov 2023 14:32:53 +0100 Subject: [PATCH 02/17] Scaffold new nodes from ItemList node --- .../nodes/ItemLists/ItemLists.node.ts | 1 + .../Transform/Aggregate/Aggregate.node.ts | 0 .../Aggregate/aggregate.operation.ts | 406 ++++++++++++ .../nodes/Transform/Limit/Limit.node.ts | 0 .../nodes/Transform/Limit/limit.operation.ts | 61 ++ .../RemoveDuplicates/RemoveDuplicates.node.ts | 0 .../removeDuplicates.operation.ts | 244 +++++++ .../nodes/Transform/Sort/Sort.node.ts | 0 .../nodes/Transform/Sort/sort.operation.ts | 275 ++++++++ .../nodes/Transform/SplitOut/SplitOut.node.ts | 0 .../SplitOut/splitOutItems.operation.ts | 248 +++++++ .../Transform/Summarize/Summarize.node.ts | 0 .../Summarize/summarize.operation.ts | 615 ++++++++++++++++++ packages/nodes-base/package.json | 8 +- 14 files changed, 1857 insertions(+), 1 deletion(-) create mode 100644 packages/nodes-base/nodes/Transform/Aggregate/Aggregate.node.ts create mode 100644 packages/nodes-base/nodes/Transform/Aggregate/aggregate.operation.ts create mode 100644 packages/nodes-base/nodes/Transform/Limit/Limit.node.ts create mode 100644 packages/nodes-base/nodes/Transform/Limit/limit.operation.ts create mode 100644 packages/nodes-base/nodes/Transform/RemoveDuplicates/RemoveDuplicates.node.ts create mode 100644 packages/nodes-base/nodes/Transform/RemoveDuplicates/removeDuplicates.operation.ts create mode 100644 packages/nodes-base/nodes/Transform/Sort/Sort.node.ts create mode 100644 packages/nodes-base/nodes/Transform/Sort/sort.operation.ts create mode 100644 packages/nodes-base/nodes/Transform/SplitOut/SplitOut.node.ts create mode 100644 packages/nodes-base/nodes/Transform/SplitOut/splitOutItems.operation.ts create mode 100644 packages/nodes-base/nodes/Transform/Summarize/Summarize.node.ts create mode 100644 packages/nodes-base/nodes/Transform/Summarize/summarize.operation.ts diff --git a/packages/nodes-base/nodes/ItemLists/ItemLists.node.ts b/packages/nodes-base/nodes/ItemLists/ItemLists.node.ts index d454c14e92cd8..1eea1f4d60591 100644 --- a/packages/nodes-base/nodes/ItemLists/ItemLists.node.ts +++ b/packages/nodes-base/nodes/ItemLists/ItemLists.node.ts @@ -12,6 +12,7 @@ export class ItemLists extends VersionedNodeType { name: 'itemLists', icon: 'file:itemLists.svg', group: ['input'], + hidden: true, subtitle: '={{$parameter["operation"] + ": " + $parameter["resource"]}}', description: 'Helper for working with lists of items and transforming arrays', defaultVersion: 3, diff --git a/packages/nodes-base/nodes/Transform/Aggregate/Aggregate.node.ts b/packages/nodes-base/nodes/Transform/Aggregate/Aggregate.node.ts new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/packages/nodes-base/nodes/Transform/Aggregate/aggregate.operation.ts b/packages/nodes-base/nodes/Transform/Aggregate/aggregate.operation.ts new file mode 100644 index 0000000000000..138fd2d2767b7 --- /dev/null +++ b/packages/nodes-base/nodes/Transform/Aggregate/aggregate.operation.ts @@ -0,0 +1,406 @@ +import type { + IDataObject, + IExecuteFunctions, + INodeExecutionData, + INodeProperties, + IPairedItemData, +} from 'n8n-workflow'; +import { NodeOperationError } from 'n8n-workflow'; + +import get from 'lodash/get'; +import isEmpty from 'lodash/isEmpty'; +import set from 'lodash/set'; + +import { addBinariesToItem, prepareFieldsArray } from '../../helpers/utils'; +import { disableDotNotationBoolean } from '../common.descriptions'; +import { updateDisplayOptions } from '@utils/utilities'; + +const properties: INodeProperties[] = [ + { + displayName: 'Aggregate', + name: 'aggregate', + type: 'options', + default: 'aggregateIndividualFields', + options: [ + { + name: 'Individual Fields', + value: 'aggregateIndividualFields', + }, + { + name: 'All Item Data (Into a Single List)', + value: 'aggregateAllItemData', + }, + ], + }, + { + displayName: 'Fields To Aggregate', + name: 'fieldsToAggregate', + type: 'fixedCollection', + typeOptions: { + multipleValues: true, + }, + placeholder: 'Add Field To Aggregate', + default: { fieldToAggregate: [{ fieldToAggregate: '', renameField: false }] }, + displayOptions: { + show: { + aggregate: ['aggregateIndividualFields'], + }, + }, + options: [ + { + displayName: '', + name: 'fieldToAggregate', + values: [ + { + displayName: 'Input Field Name', + name: 'fieldToAggregate', + type: 'string', + default: '', + description: 'The name of a field in the input items to aggregate together', + // eslint-disable-next-line n8n-nodes-base/node-param-placeholder-miscased-id + placeholder: 'e.g. id', + hint: ' Enter the field name as text', + requiresDataPath: 'single', + }, + { + displayName: 'Rename Field', + name: 'renameField', + type: 'boolean', + default: false, + description: 'Whether to give the field a different name in the output', + }, + { + displayName: 'Output Field Name', + name: 'outputFieldName', + displayOptions: { + show: { + renameField: [true], + }, + }, + type: 'string', + default: '', + description: + 'The name of the field to put the aggregated data in. Leave blank to use the input field name.', + requiresDataPath: 'single', + }, + ], + }, + ], + }, + { + displayName: 'Put Output in Field', + name: 'destinationFieldName', + type: 'string', + displayOptions: { + show: { + aggregate: ['aggregateAllItemData'], + }, + }, + default: 'data', + description: 'The name of the output field to put the data in', + }, + { + displayName: 'Include', + name: 'include', + type: 'options', + default: 'allFields', + options: [ + { + name: 'All Fields', + value: 'allFields', + }, + { + name: 'Specified Fields', + value: 'specifiedFields', + }, + { + name: 'All Fields Except', + value: 'allFieldsExcept', + }, + ], + displayOptions: { + show: { + aggregate: ['aggregateAllItemData'], + }, + }, + }, + { + displayName: 'Fields To Exclude', + name: 'fieldsToExclude', + type: 'string', + placeholder: 'e.g. email, name', + default: '', + requiresDataPath: 'multiple', + displayOptions: { + show: { + aggregate: ['aggregateAllItemData'], + include: ['allFieldsExcept'], + }, + }, + }, + { + displayName: 'Fields To Include', + name: 'fieldsToInclude', + type: 'string', + placeholder: 'e.g. email, name', + default: '', + requiresDataPath: 'multiple', + displayOptions: { + show: { + aggregate: ['aggregateAllItemData'], + include: ['specifiedFields'], + }, + }, + }, + { + displayName: 'Options', + name: 'options', + type: 'collection', + placeholder: 'Add Field', + default: {}, + options: [ + { + ...disableDotNotationBoolean, + displayOptions: { + hide: { + '/aggregate': ['aggregateAllItemData'], + }, + }, + }, + { + displayName: 'Merge Lists', + name: 'mergeLists', + type: 'boolean', + default: false, + description: + 'Whether to merge the output into a single flat list (rather than a list of lists), if the field to aggregate is a list', + displayOptions: { + hide: { + '/aggregate': ['aggregateAllItemData'], + }, + }, + }, + { + displayName: 'Include Binaries', + name: 'includeBinaries', + type: 'boolean', + default: false, + description: 'Whether to include the binary data in the new item', + }, + { + displayName: 'Keep Only Unique Binaries', + name: 'keepOnlyUnique', + type: 'boolean', + default: false, + description: + 'Whether to keep only unique binaries by comparing mime types, file types, file sizes and file extensions', + displayOptions: { + show: { + includeBinaries: [true], + }, + }, + }, + { + displayName: 'Keep Missing And Null Values', + name: 'keepMissing', + type: 'boolean', + default: false, + description: + 'Whether to add a null entry to the aggregated list when there is a missing or null value', + displayOptions: { + hide: { + '/aggregate': ['aggregateAllItemData'], + }, + }, + }, + ], + }, +]; + +const displayOptions = { + show: { + resource: ['itemList'], + operation: ['concatenateItems'], + }, +}; + +export const description = updateDisplayOptions(displayOptions, properties); + +export async function execute( + this: IExecuteFunctions, + items: INodeExecutionData[], +): Promise { + let returnData: INodeExecutionData = { json: {}, pairedItem: [] }; + + const aggregate = this.getNodeParameter('aggregate', 0, '') as string; + + if (aggregate === 'aggregateIndividualFields') { + const disableDotNotation = this.getNodeParameter( + 'options.disableDotNotation', + 0, + false, + ) as boolean; + const mergeLists = this.getNodeParameter('options.mergeLists', 0, false) as boolean; + const fieldsToAggregate = this.getNodeParameter( + 'fieldsToAggregate.fieldToAggregate', + 0, + [], + ) as [{ fieldToAggregate: string; renameField: boolean; outputFieldName: string }]; + const keepMissing = this.getNodeParameter('options.keepMissing', 0, false) as boolean; + + if (!fieldsToAggregate.length) { + throw new NodeOperationError(this.getNode(), 'No fields specified', { + description: 'Please add a field to aggregate', + }); + } + + const newItem: INodeExecutionData = { + json: {}, + pairedItem: Array.from({ length: items.length }, (_, i) => i).map((index) => { + return { + item: index, + }; + }), + }; + + const values: { [key: string]: any } = {}; + const outputFields: string[] = []; + + for (const { fieldToAggregate, outputFieldName, renameField } of fieldsToAggregate) { + const field = renameField ? outputFieldName : fieldToAggregate; + + if (outputFields.includes(field)) { + throw new NodeOperationError( + this.getNode(), + `The '${field}' output field is used more than once`, + { description: 'Please make sure each output field name is unique' }, + ); + } else { + outputFields.push(field); + } + + const getFieldToAggregate = () => + !disableDotNotation && fieldToAggregate.includes('.') + ? fieldToAggregate.split('.').pop() + : fieldToAggregate; + + const _outputFieldName = outputFieldName + ? outputFieldName + : (getFieldToAggregate() as string); + + if (fieldToAggregate !== '') { + values[_outputFieldName] = []; + for (let i = 0; i < items.length; i++) { + if (!disableDotNotation) { + let value = get(items[i].json, fieldToAggregate); + + if (!keepMissing) { + if (Array.isArray(value)) { + value = value.filter((entry) => entry !== null); + } else if (value === null || value === undefined) { + continue; + } + } + + if (Array.isArray(value) && mergeLists) { + values[_outputFieldName].push(...value); + } else { + values[_outputFieldName].push(value); + } + } else { + let value = items[i].json[fieldToAggregate]; + + if (!keepMissing) { + if (Array.isArray(value)) { + value = value.filter((entry) => entry !== null); + } else if (value === null || value === undefined) { + continue; + } + } + + if (Array.isArray(value) && mergeLists) { + values[_outputFieldName].push(...value); + } else { + values[_outputFieldName].push(value); + } + } + } + } + } + + for (const key of Object.keys(values)) { + if (!disableDotNotation) { + set(newItem.json, key, values[key]); + } else { + newItem.json[key] = values[key]; + } + } + + returnData = newItem; + } else { + let newItems: IDataObject[] = items.map((item) => item.json); + let pairedItem: IPairedItemData[] = []; + const destinationFieldName = this.getNodeParameter('destinationFieldName', 0) as string; + + const fieldsToExclude = prepareFieldsArray( + this.getNodeParameter('fieldsToExclude', 0, '') as string, + 'Fields To Exclude', + ); + + const fieldsToInclude = prepareFieldsArray( + this.getNodeParameter('fieldsToInclude', 0, '') as string, + 'Fields To Include', + ); + + if (fieldsToExclude.length || fieldsToInclude.length) { + newItems = newItems.reduce((acc, item, index) => { + const newItem: IDataObject = {}; + let outputFields = Object.keys(item); + + if (fieldsToExclude.length) { + outputFields = outputFields.filter((key) => !fieldsToExclude.includes(key)); + } + if (fieldsToInclude.length) { + outputFields = outputFields.filter((key) => + fieldsToInclude.length ? fieldsToInclude.includes(key) : true, + ); + } + + outputFields.forEach((key) => { + newItem[key] = item[key]; + }); + + if (isEmpty(newItem)) { + return acc; + } + + pairedItem.push({ item: index }); + return acc.concat([newItem]); + }, [] as IDataObject[]); + } else { + pairedItem = Array.from({ length: newItems.length }, (_, item) => ({ + item, + })); + } + + const output: INodeExecutionData = { json: { [destinationFieldName]: newItems }, pairedItem }; + + returnData = output; + } + + const includeBinaries = this.getNodeParameter('options.includeBinaries', 0, false) as boolean; + + if (includeBinaries) { + const pairedItems = (returnData.pairedItem || []) as IPairedItemData[]; + + const aggregatedItems = pairedItems.map((item) => { + return items[item.item]; + }); + + const keepOnlyUnique = this.getNodeParameter('options.keepOnlyUnique', 0, false) as boolean; + + addBinariesToItem(returnData, aggregatedItems, keepOnlyUnique); + } + + return [returnData]; +} diff --git a/packages/nodes-base/nodes/Transform/Limit/Limit.node.ts b/packages/nodes-base/nodes/Transform/Limit/Limit.node.ts new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/packages/nodes-base/nodes/Transform/Limit/limit.operation.ts b/packages/nodes-base/nodes/Transform/Limit/limit.operation.ts new file mode 100644 index 0000000000000..5671cf246fa03 --- /dev/null +++ b/packages/nodes-base/nodes/Transform/Limit/limit.operation.ts @@ -0,0 +1,61 @@ +import type { IExecuteFunctions, INodeExecutionData, INodeProperties } from 'n8n-workflow'; +import { updateDisplayOptions } from '@utils/utilities'; + +const properties: INodeProperties[] = [ + { + displayName: 'Max Items', + name: 'maxItems', + type: 'number', + typeOptions: { + minValue: 1, + }, + default: 1, + description: 'If there are more items than this number, some are removed', + }, + { + displayName: 'Keep', + name: 'keep', + type: 'options', + options: [ + { + name: 'First Items', + value: 'firstItems', + }, + { + name: 'Last Items', + value: 'lastItems', + }, + ], + default: 'firstItems', + description: 'When removing items, whether to keep the ones at the start or the ending', + }, +]; + +const displayOptions = { + show: { + resource: ['itemList'], + operation: ['limit'], + }, +}; + +export const description = updateDisplayOptions(displayOptions, properties); + +export async function execute( + this: IExecuteFunctions, + items: INodeExecutionData[], +): Promise { + let returnData = items; + const maxItems = this.getNodeParameter('maxItems', 0) as number; + const keep = this.getNodeParameter('keep', 0) as string; + + if (maxItems > items.length) { + return returnData; + } + + if (keep === 'firstItems') { + returnData = items.slice(0, maxItems); + } else { + returnData = items.slice(items.length - maxItems, items.length); + } + return returnData; +} diff --git a/packages/nodes-base/nodes/Transform/RemoveDuplicates/RemoveDuplicates.node.ts b/packages/nodes-base/nodes/Transform/RemoveDuplicates/RemoveDuplicates.node.ts new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/packages/nodes-base/nodes/Transform/RemoveDuplicates/removeDuplicates.operation.ts b/packages/nodes-base/nodes/Transform/RemoveDuplicates/removeDuplicates.operation.ts new file mode 100644 index 0000000000000..e97be1359ae79 --- /dev/null +++ b/packages/nodes-base/nodes/Transform/RemoveDuplicates/removeDuplicates.operation.ts @@ -0,0 +1,244 @@ +import type { IExecuteFunctions, INodeExecutionData, INodeProperties } from 'n8n-workflow'; +import { NodeOperationError } from 'n8n-workflow'; + +import get from 'lodash/get'; +import isEqual from 'lodash/isEqual'; +import lt from 'lodash/lt'; +import pick from 'lodash/pick'; + +import { compareItems, flattenKeys, prepareFieldsArray } from '../../helpers/utils'; +import { disableDotNotationBoolean } from '../common.descriptions'; +import { updateDisplayOptions } from '@utils/utilities'; + +const properties: INodeProperties[] = [ + { + displayName: 'Compare', + name: 'compare', + type: 'options', + options: [ + { + name: 'All Fields', + value: 'allFields', + }, + { + name: 'All Fields Except', + value: 'allFieldsExcept', + }, + { + name: 'Selected Fields', + value: 'selectedFields', + }, + ], + default: 'allFields', + description: 'The fields of the input items to compare to see if they are the same', + }, + { + displayName: 'Fields To Exclude', + name: 'fieldsToExclude', + type: 'string', + placeholder: 'e.g. email, name', + requiresDataPath: 'multiple', + description: 'Fields in the input to exclude from the comparison', + default: '', + displayOptions: { + show: { + compare: ['allFieldsExcept'], + }, + }, + }, + { + displayName: 'Fields To Compare', + name: 'fieldsToCompare', + type: 'string', + placeholder: 'e.g. email, name', + requiresDataPath: 'multiple', + description: 'Fields in the input to add to the comparison', + default: '', + displayOptions: { + show: { + compare: ['selectedFields'], + }, + }, + }, + { + displayName: 'Options', + name: 'options', + type: 'collection', + placeholder: 'Add Field', + default: {}, + displayOptions: { + show: { + compare: ['allFieldsExcept', 'selectedFields'], + }, + }, + options: [ + disableDotNotationBoolean, + { + displayName: 'Remove Other Fields', + name: 'removeOtherFields', + type: 'boolean', + default: false, + description: + 'Whether to remove any fields that are not being compared. If disabled, will keep the values from the first of the duplicates.', + }, + ], + }, +]; + +const displayOptions = { + show: { + resource: ['itemList'], + operation: ['removeDuplicates'], + }, +}; + +export const description = updateDisplayOptions(displayOptions, properties); + +export async function execute( + this: IExecuteFunctions, + items: INodeExecutionData[], +): Promise { + const compare = this.getNodeParameter('compare', 0) as string; + const disableDotNotation = this.getNodeParameter( + 'options.disableDotNotation', + 0, + false, + ) as boolean; + const removeOtherFields = this.getNodeParameter('options.removeOtherFields', 0, false) as boolean; + + let keys = disableDotNotation + ? Object.keys(items[0].json) + : Object.keys(flattenKeys(items[0].json)); + + for (const item of items) { + for (const key of disableDotNotation + ? Object.keys(item.json) + : Object.keys(flattenKeys(item.json))) { + if (!keys.includes(key)) { + keys.push(key); + } + } + } + + if (compare === 'allFieldsExcept') { + const fieldsToExclude = prepareFieldsArray( + this.getNodeParameter('fieldsToExclude', 0, '') as string, + 'Fields To Exclude', + ); + + if (!fieldsToExclude.length) { + throw new NodeOperationError( + this.getNode(), + 'No fields specified. Please add a field to exclude from comparison', + ); + } + if (!disableDotNotation) { + keys = Object.keys(flattenKeys(items[0].json)); + } + keys = keys.filter((key) => !fieldsToExclude.includes(key)); + } + if (compare === 'selectedFields') { + const fieldsToCompare = prepareFieldsArray( + this.getNodeParameter('fieldsToCompare', 0, '') as string, + 'Fields To Compare', + ); + if (!fieldsToCompare.length) { + throw new NodeOperationError( + this.getNode(), + 'No fields specified. Please add a field to compare on', + ); + } + if (!disableDotNotation) { + keys = Object.keys(flattenKeys(items[0].json)); + } + keys = fieldsToCompare.map((key) => key.trim()); + } + + // This solution is O(nlogn) + // add original index to the items + const newItems = items.map( + (item, index) => + ({ + json: { ...item.json, __INDEX: index }, + pairedItem: { item: index }, + }) as INodeExecutionData, + ); + //sort items using the compare keys + newItems.sort((a, b) => { + let result = 0; + + for (const key of keys) { + let equal; + if (!disableDotNotation) { + equal = isEqual(get(a.json, key), get(b.json, key)); + } else { + equal = isEqual(a.json[key], b.json[key]); + } + if (!equal) { + let lessThan; + if (!disableDotNotation) { + lessThan = lt(get(a.json, key), get(b.json, key)); + } else { + lessThan = lt(a.json[key], b.json[key]); + } + result = lessThan ? -1 : 1; + break; + } + } + return result; + }); + + for (const key of keys) { + let type: any = undefined; + for (const item of newItems) { + if (key === '') { + throw new NodeOperationError(this.getNode(), 'Name of field to compare is blank'); + } + const value = !disableDotNotation ? get(item.json, key) : item.json[key]; + if (value === undefined && disableDotNotation && key.includes('.')) { + throw new NodeOperationError( + this.getNode(), + `'${key}' field is missing from some input items`, + { + description: + "If you're trying to use a nested field, make sure you turn off 'disable dot notation' in the node options", + }, + ); + } else if (value === undefined) { + throw new NodeOperationError( + this.getNode(), + `'${key}' field is missing from some input items`, + ); + } + if (type !== undefined && value !== undefined && type !== typeof value) { + throw new NodeOperationError(this.getNode(), `'${key}' isn't always the same type`, { + description: 'The type of this field varies between items', + }); + } else { + type = typeof value; + } + } + } + + // collect the original indexes of items to be removed + const removedIndexes: number[] = []; + let temp = newItems[0]; + for (let index = 1; index < newItems.length; index++) { + if (compareItems(newItems[index], temp, keys, disableDotNotation, this.getNode())) { + removedIndexes.push(newItems[index].json.__INDEX as unknown as number); + } else { + temp = newItems[index]; + } + } + + let returnData = items.filter((_, index) => !removedIndexes.includes(index)); + + if (removeOtherFields) { + returnData = returnData.map((item, index) => ({ + json: pick(item.json, ...keys), + pairedItem: { item: index }, + })); + } + + return returnData; +} diff --git a/packages/nodes-base/nodes/Transform/Sort/Sort.node.ts b/packages/nodes-base/nodes/Transform/Sort/Sort.node.ts new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/packages/nodes-base/nodes/Transform/Sort/sort.operation.ts b/packages/nodes-base/nodes/Transform/Sort/sort.operation.ts new file mode 100644 index 0000000000000..6825779e28a99 --- /dev/null +++ b/packages/nodes-base/nodes/Transform/Sort/sort.operation.ts @@ -0,0 +1,275 @@ +import type { + IExecuteFunctions, + IDataObject, + INodeExecutionData, + INodeProperties, +} from 'n8n-workflow'; +import { NodeOperationError } from 'n8n-workflow'; + +import get from 'lodash/get'; + +import isEqual from 'lodash/isEqual'; +import lt from 'lodash/lt'; + +import { shuffleArray, sortByCode } from '../../helpers/utils'; +import { disableDotNotationBoolean } from '../common.descriptions'; +import { updateDisplayOptions } from '@utils/utilities'; + +const properties: INodeProperties[] = [ + { + displayName: 'Type', + name: 'type', + type: 'options', + options: [ + { + name: 'Simple', + value: 'simple', + }, + { + name: 'Random', + value: 'random', + }, + { + name: 'Code', + value: 'code', + }, + ], + default: 'simple', + description: 'The fields of the input items to compare to see if they are the same', + }, + { + displayName: 'Fields To Sort By', + name: 'sortFieldsUi', + type: 'fixedCollection', + typeOptions: { + multipleValues: true, + }, + placeholder: 'Add Field To Sort By', + options: [ + { + displayName: '', + name: 'sortField', + values: [ + { + displayName: 'Field Name', + name: 'fieldName', + type: 'string', + required: true, + default: '', + description: 'The field to sort by', + // eslint-disable-next-line n8n-nodes-base/node-param-placeholder-miscased-id + placeholder: 'e.g. id', + hint: ' Enter the field name as text', + requiresDataPath: 'single', + }, + { + displayName: 'Order', + name: 'order', + type: 'options', + options: [ + { + name: 'Ascending', + value: 'ascending', + }, + { + name: 'Descending', + value: 'descending', + }, + ], + default: 'ascending', + description: 'The order to sort by', + }, + ], + }, + ], + default: {}, + description: 'The fields of the input items to compare to see if they are the same', + displayOptions: { + show: { + type: ['simple'], + }, + }, + }, + { + displayName: 'Code', + name: 'code', + type: 'string', + typeOptions: { + alwaysOpenEditWindow: true, + editor: 'code', + rows: 10, + }, + default: `// The two items to compare are in the variables a and b +// Access the fields in a.json and b.json +// Return -1 if a should go before b +// Return 1 if b should go before a +// Return 0 if there's no difference + +fieldName = 'myField'; + +if (a.json[fieldName] < b.json[fieldName]) { +return -1; +} +if (a.json[fieldName] > b.json[fieldName]) { +return 1; +} +return 0;`, + description: 'Javascript code to determine the order of any two items', + displayOptions: { + show: { + type: ['code'], + }, + }, + }, + { + displayName: 'Options', + name: 'options', + type: 'collection', + placeholder: 'Add Field', + default: {}, + displayOptions: { + show: { + type: ['simple'], + }, + }, + options: [disableDotNotationBoolean], + }, +]; + +const displayOptions = { + show: { + resource: ['itemList'], + operation: ['sort'], + }, +}; + +export const description = updateDisplayOptions(displayOptions, properties); + +export async function execute( + this: IExecuteFunctions, + items: INodeExecutionData[], +): Promise { + let returnData = [...items]; + const type = this.getNodeParameter('type', 0) as string; + const disableDotNotation = this.getNodeParameter( + 'options.disableDotNotation', + 0, + false, + ) as boolean; + + if (type === 'random') { + shuffleArray(returnData); + return returnData; + } + + if (type === 'simple') { + const sortFieldsUi = this.getNodeParameter('sortFieldsUi', 0) as IDataObject; + const sortFields = sortFieldsUi.sortField as Array<{ + fieldName: string; + order: 'ascending' | 'descending'; + }>; + + if (!sortFields?.length) { + throw new NodeOperationError( + this.getNode(), + 'No sorting specified. Please add a field to sort by', + ); + } + + for (const { fieldName } of sortFields) { + let found = false; + for (const item of items) { + if (!disableDotNotation) { + if (get(item.json, fieldName) !== undefined) { + found = true; + } + } else if (item.json.hasOwnProperty(fieldName)) { + found = true; + } + } + if (!found && disableDotNotation && fieldName.includes('.')) { + throw new NodeOperationError( + this.getNode(), + `Couldn't find the field '${fieldName}' in the input data`, + { + description: + "If you're trying to use a nested field, make sure you turn off 'disable dot notation' in the node options", + }, + ); + } else if (!found) { + throw new NodeOperationError( + this.getNode(), + `Couldn't find the field '${fieldName}' in the input data`, + ); + } + } + + const sortFieldsWithDirection = sortFields.map((field) => ({ + name: field.fieldName, + dir: field.order === 'ascending' ? 1 : -1, + })); + + returnData.sort((a, b) => { + let result = 0; + for (const field of sortFieldsWithDirection) { + let equal; + if (!disableDotNotation) { + const _a = + typeof get(a.json, field.name) === 'string' + ? (get(a.json, field.name) as string).toLowerCase() + : get(a.json, field.name); + const _b = + typeof get(b.json, field.name) === 'string' + ? (get(b.json, field.name) as string).toLowerCase() + : get(b.json, field.name); + equal = isEqual(_a, _b); + } else { + const _a = + typeof a.json[field.name] === 'string' + ? (a.json[field.name] as string).toLowerCase() + : a.json[field.name]; + const _b = + typeof b.json[field.name] === 'string' + ? (b.json[field.name] as string).toLowerCase() + : b.json[field.name]; + equal = isEqual(_a, _b); + } + + if (!equal) { + let lessThan; + if (!disableDotNotation) { + const _a = + typeof get(a.json, field.name) === 'string' + ? (get(a.json, field.name) as string).toLowerCase() + : get(a.json, field.name); + const _b = + typeof get(b.json, field.name) === 'string' + ? (get(b.json, field.name) as string).toLowerCase() + : get(b.json, field.name); + lessThan = lt(_a, _b); + } else { + const _a = + typeof a.json[field.name] === 'string' + ? (a.json[field.name] as string).toLowerCase() + : a.json[field.name]; + const _b = + typeof b.json[field.name] === 'string' + ? (b.json[field.name] as string).toLowerCase() + : b.json[field.name]; + lessThan = lt(_a, _b); + } + if (lessThan) { + result = -1 * field.dir; + } else { + result = 1 * field.dir; + } + break; + } + } + return result; + }); + } else { + returnData = sortByCode.call(this, returnData); + } + return returnData; +} diff --git a/packages/nodes-base/nodes/Transform/SplitOut/SplitOut.node.ts b/packages/nodes-base/nodes/Transform/SplitOut/SplitOut.node.ts new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/packages/nodes-base/nodes/Transform/SplitOut/splitOutItems.operation.ts b/packages/nodes-base/nodes/Transform/SplitOut/splitOutItems.operation.ts new file mode 100644 index 0000000000000..13d5bd89189b2 --- /dev/null +++ b/packages/nodes-base/nodes/Transform/SplitOut/splitOutItems.operation.ts @@ -0,0 +1,248 @@ +import type { + IBinaryData, + IDataObject, + IExecuteFunctions, + INodeExecutionData, + INodeProperties, +} from 'n8n-workflow'; +import { deepCopy, NodeOperationError } from 'n8n-workflow'; + +import get from 'lodash/get'; +import unset from 'lodash/unset'; +import { disableDotNotationBoolean } from '../common.descriptions'; +import { prepareFieldsArray } from '../../helpers/utils'; +import { updateDisplayOptions } from '@utils/utilities'; + +const properties: INodeProperties[] = [ + { + displayName: 'Fields To Split Out', + name: 'fieldToSplitOut', + type: 'string', + default: '', + required: true, + placeholder: 'Drag fields from the left or type their names', + description: + 'The name of the input fields to break out into separate items. Separate multiple field names by commas. For binary data, use $binary.', + requiresDataPath: 'multiple', + }, + { + displayName: 'Include', + name: 'include', + type: 'options', + options: [ + { + name: 'No Other Fields', + value: 'noOtherFields', + }, + { + name: 'All Other Fields', + value: 'allOtherFields', + }, + { + name: 'Selected Other Fields', + value: 'selectedOtherFields', + }, + ], + default: 'noOtherFields', + description: 'Whether to copy any other fields into the new items', + }, + { + displayName: 'Fields To Include', + name: 'fieldsToInclude', + type: 'string', + placeholder: 'e.g. email, name', + requiresDataPath: 'multiple', + description: 'Fields in the input items to aggregate together', + default: '', + displayOptions: { + show: { + include: ['selectedOtherFields'], + }, + }, + }, + { + displayName: 'Options', + name: 'options', + type: 'collection', + placeholder: 'Add Field', + default: {}, + options: [ + disableDotNotationBoolean, + { + displayName: 'Destination Field Name', + name: 'destinationFieldName', + type: 'string', + requiresDataPath: 'multiple', + default: '', + description: 'The field in the output under which to put the split field contents', + }, + { + displayName: 'Include Binary', + name: 'includeBinary', + type: 'boolean', + default: false, + description: 'Whether to include the binary data in the new items', + }, + ], + }, +]; + +const displayOptions = { + show: { + resource: ['itemList'], + operation: ['splitOutItems'], + }, +}; + +export const description = updateDisplayOptions(displayOptions, properties); + +export async function execute( + this: IExecuteFunctions, + items: INodeExecutionData[], +): Promise { + const returnData: INodeExecutionData[] = []; + + for (let i = 0; i < items.length; i++) { + const fieldsToSplitOut = (this.getNodeParameter('fieldToSplitOut', i) as string) + .split(',') + .map((field) => field.trim().replace(/^\$json\./, '')); + + const options = this.getNodeParameter('options', i, {}); + + const disableDotNotation = options.disableDotNotation as boolean; + + const destinationFields = ((options.destinationFieldName as string) || '') + .split(',') + .filter((field) => field.trim() !== '') + .map((field) => field.trim()); + + if (destinationFields.length && destinationFields.length !== fieldsToSplitOut.length) { + throw new NodeOperationError( + this.getNode(), + 'If multiple fields to split out are given, the same number of destination fields must be given', + ); + } + + const include = this.getNodeParameter('include', i) as + | 'selectedOtherFields' + | 'allOtherFields' + | 'noOtherFields'; + + const multiSplit = fieldsToSplitOut.length > 1; + + const item = { ...items[i].json }; + const splited: INodeExecutionData[] = []; + for (const [entryIndex, fieldToSplitOut] of fieldsToSplitOut.entries()) { + const destinationFieldName = destinationFields[entryIndex] || ''; + + let entityToSplit: IDataObject[] = []; + + if (fieldToSplitOut === '$binary') { + entityToSplit = Object.entries(items[i].binary || {}).map(([key, value]) => ({ + [key]: value, + })); + } else { + if (!disableDotNotation) { + entityToSplit = get(item, fieldToSplitOut) as IDataObject[]; + } else { + entityToSplit = item[fieldToSplitOut] as IDataObject[]; + } + + if (entityToSplit === undefined) { + entityToSplit = []; + } + + if (typeof entityToSplit !== 'object' || entityToSplit === null) { + entityToSplit = [entityToSplit]; + } + + if (!Array.isArray(entityToSplit)) { + entityToSplit = Object.values(entityToSplit); + } + } + + for (const [elementIndex, element] of entityToSplit.entries()) { + if (splited[elementIndex] === undefined) { + splited[elementIndex] = { json: {}, pairedItem: { item: i } }; + } + + const fieldName = destinationFieldName || fieldToSplitOut; + + if (fieldToSplitOut === '$binary') { + if (splited[elementIndex].binary === undefined) { + splited[elementIndex].binary = {}; + } + splited[elementIndex].binary![Object.keys(element)[0]] = Object.values( + element, + )[0] as IBinaryData; + + continue; + } + + if (typeof element === 'object' && element !== null && include === 'noOtherFields') { + if (destinationFieldName === '' && !multiSplit) { + splited[elementIndex] = { + json: { ...splited[elementIndex].json, ...element }, + pairedItem: { item: i }, + }; + } else { + splited[elementIndex].json[fieldName] = element; + } + } else { + splited[elementIndex].json[fieldName] = element; + } + } + } + + for (const splitEntry of splited) { + let newItem: INodeExecutionData = splitEntry; + + if (include === 'allOtherFields') { + const itemCopy = deepCopy(item); + for (const fieldToSplitOut of fieldsToSplitOut) { + if (!disableDotNotation) { + unset(itemCopy, fieldToSplitOut); + } else { + delete itemCopy[fieldToSplitOut]; + } + } + newItem.json = { ...itemCopy, ...splitEntry.json }; + } + + if (include === 'selectedOtherFields') { + const fieldsToInclude = prepareFieldsArray( + this.getNodeParameter('fieldsToInclude', i, '') as string, + 'Fields To Include', + ); + + if (!fieldsToInclude.length) { + throw new NodeOperationError(this.getNode(), 'No fields specified', { + description: 'Please add a field to include', + }); + } + + for (const field of fieldsToInclude) { + if (!disableDotNotation) { + splitEntry.json[field] = get(item, field); + } else { + splitEntry.json[field] = item[field]; + } + } + + newItem = splitEntry; + } + + const includeBinary = options.includeBinary as boolean; + + if (includeBinary) { + if (items[i].binary && !newItem.binary) { + newItem.binary = items[i].binary; + } + } + + returnData.push(newItem); + } + } + + return returnData; +} diff --git a/packages/nodes-base/nodes/Transform/Summarize/Summarize.node.ts b/packages/nodes-base/nodes/Transform/Summarize/Summarize.node.ts new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/packages/nodes-base/nodes/Transform/Summarize/summarize.operation.ts b/packages/nodes-base/nodes/Transform/Summarize/summarize.operation.ts new file mode 100644 index 0000000000000..059446ac0f19a --- /dev/null +++ b/packages/nodes-base/nodes/Transform/Summarize/summarize.operation.ts @@ -0,0 +1,615 @@ +import type { + GenericValue, + IDataObject, + IExecuteFunctions, + INodeExecutionData, + INodeProperties, +} from 'n8n-workflow'; +import { NodeOperationError } from 'n8n-workflow'; + +import get from 'lodash/get'; +import { disableDotNotationBoolean } from '../common.descriptions'; +import { updateDisplayOptions } from '@utils/utilities'; + +type AggregationType = + | 'append' + | 'average' + | 'concatenate' + | 'count' + | 'countUnique' + | 'max' + | 'min' + | 'sum'; + +type Aggregation = { + aggregation: AggregationType; + field: string; + includeEmpty?: boolean; + separateBy?: string; + customSeparator?: string; +}; + +type Aggregations = Aggregation[]; + +const AggregationDisplayNames = { + append: 'appended_', + average: 'average_', + concatenate: 'concatenated_', + count: 'count_', + countUnique: 'unique_count_', + max: 'max_', + min: 'min_', + sum: 'sum_', +}; + +const NUMERICAL_AGGREGATIONS = ['average', 'sum']; + +type SummarizeOptions = { + disableDotNotation?: boolean; + outputFormat?: 'separateItems' | 'singleItem'; + skipEmptySplitFields?: boolean; +}; + +type ValueGetterFn = ( + item: IDataObject, + field: string, +) => IDataObject | IDataObject[] | GenericValue | GenericValue[]; + +export const properties: INodeProperties[] = [ + { + displayName: 'Fields to Summarize', + name: 'fieldsToSummarize', + type: 'fixedCollection', + placeholder: 'Add Field', + default: { values: [{ aggregation: 'count', field: '' }] }, + typeOptions: { + multipleValues: true, + }, + options: [ + { + displayName: '', + name: 'values', + values: [ + { + displayName: 'Aggregation', + name: 'aggregation', + type: 'options', + options: [ + { + name: 'Append', + value: 'append', + }, + { + name: 'Average', + value: 'average', + }, + { + name: 'Concatenate', + value: 'concatenate', + }, + { + name: 'Count', + value: 'count', + }, + { + name: 'Count Unique', + value: 'countUnique', + }, + { + name: 'Max', + value: 'max', + }, + { + name: 'Min', + value: 'min', + }, + { + name: 'Sum', + value: 'sum', + }, + ], + default: 'count', + description: 'How to combine the values of the field you want to summarize', + }, + //field repeated to have different descriptions for different aggregations -------------------------------- + { + displayName: 'Field', + name: 'field', + type: 'string', + default: '', + description: 'The name of an input field that you want to summarize', + placeholder: 'e.g. cost', + hint: ' Enter the field name as text', + displayOptions: { + hide: { + aggregation: [...NUMERICAL_AGGREGATIONS, 'countUnique', 'count', 'max', 'min'], + }, + }, + requiresDataPath: 'single', + }, + { + displayName: 'Field', + name: 'field', + type: 'string', + default: '', + description: + 'The name of an input field that you want to summarize. The field should contain numerical values; null, undefined, empty strings would be ignored.', + placeholder: 'e.g. cost', + hint: ' Enter the field name as text', + displayOptions: { + show: { + aggregation: NUMERICAL_AGGREGATIONS, + }, + }, + requiresDataPath: 'single', + }, + { + displayName: 'Field', + name: 'field', + type: 'string', + default: '', + description: + 'The name of an input field that you want to summarize; null, undefined, empty strings would be ignored', + placeholder: 'e.g. cost', + hint: ' Enter the field name as text', + displayOptions: { + show: { + aggregation: ['countUnique', 'count', 'max', 'min'], + }, + }, + requiresDataPath: 'single', + }, + // ---------------------------------------------------------------------------------------------------------- + { + displayName: 'Include Empty Values', + name: 'includeEmpty', + type: 'boolean', + default: false, + displayOptions: { + show: { + aggregation: ['append', 'concatenate'], + }, + }, + }, + { + displayName: 'Separator', + name: 'separateBy', + type: 'options', + default: ',', + // eslint-disable-next-line n8n-nodes-base/node-param-options-type-unsorted-items + options: [ + { + name: 'Comma', + value: ',', + }, + { + name: 'Comma and Space', + value: ', ', + }, + { + name: 'New Line', + value: '\n', + }, + { + name: 'None', + value: '', + }, + { + name: 'Space', + value: ' ', + }, + { + name: 'Other', + value: 'other', + }, + ], + hint: 'What to insert between values', + displayOptions: { + show: { + aggregation: ['concatenate'], + }, + }, + }, + { + displayName: 'Custom Separator', + name: 'customSeparator', + type: 'string', + default: '', + displayOptions: { + show: { + aggregation: ['concatenate'], + separateBy: ['other'], + }, + }, + }, + ], + }, + ], + }, + // fieldsToSplitBy repeated to have different displayName for singleItem and separateItems ----------------------------- + { + displayName: 'Fields to Split By', + name: 'fieldsToSplitBy', + type: 'string', + placeholder: 'e.g. country, city', + default: '', + description: 'The name of the input fields that you want to split the summary by', + hint: 'Enter the name of the fields as text (separated by commas)', + displayOptions: { + hide: { + '/options.outputFormat': ['singleItem'], + }, + }, + requiresDataPath: 'multiple', + }, + { + displayName: 'Fields to Group By', + name: 'fieldsToSplitBy', + type: 'string', + placeholder: 'e.g. country, city', + default: '', + description: 'The name of the input fields that you want to split the summary by', + hint: 'Enter the name of the fields as text (separated by commas)', + displayOptions: { + show: { + '/options.outputFormat': ['singleItem'], + }, + }, + requiresDataPath: 'multiple', + }, + // ---------------------------------------------------------------------------------------------------------- + { + displayName: 'Options', + name: 'options', + type: 'collection', + placeholder: 'Add Option', + default: {}, + options: [ + disableDotNotationBoolean, + { + displayName: 'Output Format', + name: 'outputFormat', + type: 'options', + default: 'separateItems', + options: [ + { + name: 'Each Split in a Separate Item', + value: 'separateItems', + }, + { + name: 'All Splits in a Single Item', + value: 'singleItem', + }, + ], + }, + { + // eslint-disable-next-line n8n-nodes-base/node-param-display-name-miscased + displayName: 'Ignore items without valid fields to group by', + name: 'skipEmptySplitFields', + type: 'boolean', + default: false, + }, + ], + }, +]; + +const displayOptions = { + show: { + resource: ['itemList'], + operation: ['summarize'], + }, +}; + +export const description = updateDisplayOptions(displayOptions, properties); + +function isEmpty(value: T) { + return value === undefined || value === null || value === ''; +} + +function parseReturnData(returnData: IDataObject) { + const regexBrackets = /[\]\["]/g; + const regexSpaces = /[ .]/g; + for (const key of Object.keys(returnData)) { + if (key.match(regexBrackets)) { + const newKey = key.replace(regexBrackets, ''); + returnData[newKey] = returnData[key]; + delete returnData[key]; + } + } + for (const key of Object.keys(returnData)) { + if (key.match(regexSpaces)) { + const newKey = key.replace(regexSpaces, '_'); + returnData[newKey] = returnData[key]; + delete returnData[key]; + } + } +} + +function parseFieldName(fieldName: string[]) { + const regexBrackets = /[\]\["]/g; + const regexSpaces = /[ .]/g; + fieldName = fieldName.map((field) => { + field = field.replace(regexBrackets, ''); + field = field.replace(regexSpaces, '_'); + return field; + }); + return fieldName; +} + +const fieldValueGetter = (disableDotNotation?: boolean) => { + if (disableDotNotation) { + return (item: IDataObject, field: string) => item[field]; + } else { + return (item: IDataObject, field: string) => get(item, field); + } +}; + +function checkIfFieldExists( + this: IExecuteFunctions, + items: IDataObject[], + aggregations: Aggregations, + getValue: ValueGetterFn, +) { + for (const aggregation of aggregations) { + if (aggregation.field === '') { + continue; + } + const exist = items.some((item) => getValue(item, aggregation.field) !== undefined); + if (!exist) { + throw new NodeOperationError( + this.getNode(), + `The field '${aggregation.field}' does not exist in any items`, + ); + } + } +} + +function aggregate(items: IDataObject[], entry: Aggregation, getValue: ValueGetterFn) { + const { aggregation, field } = entry; + let data = [...items]; + + if (NUMERICAL_AGGREGATIONS.includes(aggregation)) { + data = data.filter( + (item) => typeof getValue(item, field) === 'number' && !isEmpty(getValue(item, field)), + ); + } + + switch (aggregation) { + //combine operations + case 'append': + if (!entry.includeEmpty) { + data = data.filter((item) => !isEmpty(getValue(item, field))); + } + return data.map((item) => getValue(item, field)); + case 'concatenate': + const separateBy = entry.separateBy === 'other' ? entry.customSeparator : entry.separateBy; + if (!entry.includeEmpty) { + data = data.filter((item) => !isEmpty(getValue(item, field))); + } + return data + .map((item) => { + let value = getValue(item, field); + if (typeof value === 'object') { + value = JSON.stringify(value); + } + if (typeof value === 'undefined') { + value = 'undefined'; + } + + return value; + }) + .join(separateBy); + + //numerical operations + case 'average': + return ( + data.reduce((acc, item) => { + return acc + (getValue(item, field) as number); + }, 0) / data.length + ); + case 'sum': + return data.reduce((acc, item) => { + return acc + (getValue(item, field) as number); + }, 0); + //comparison operations + case 'min': + let min; + for (const item of data) { + const value = getValue(item, field); + if (value !== undefined && value !== null && value !== '') { + if (min === undefined || value < min) { + min = value; + } + } + } + return min !== undefined ? min : null; + case 'max': + let max; + for (const item of data) { + const value = getValue(item, field); + if (value !== undefined && value !== null && value !== '') { + if (max === undefined || value > max) { + max = value; + } + } + } + return max !== undefined ? max : null; + + //count operations + case 'countUnique': + return new Set(data.map((item) => getValue(item, field)).filter((item) => !isEmpty(item))) + .size; + default: + //count by default + return data.filter((item) => !isEmpty(getValue(item, field))).length; + } +} + +function aggregateData( + data: IDataObject[], + fieldsToSummarize: Aggregations, + options: SummarizeOptions, + getValue: ValueGetterFn, +) { + const returnData = fieldsToSummarize.reduce((acc, aggregation) => { + acc[`${AggregationDisplayNames[aggregation.aggregation]}${aggregation.field}`] = aggregate( + data, + aggregation, + getValue, + ); + return acc; + }, {} as IDataObject); + parseReturnData(returnData); + if (options.outputFormat === 'singleItem') { + parseReturnData(returnData); + return returnData; + } else { + return { ...returnData, pairedItems: data.map((item) => item._itemIndex as number) }; + } +} + +function splitData( + splitKeys: string[], + data: IDataObject[], + fieldsToSummarize: Aggregations, + options: SummarizeOptions, + getValue: ValueGetterFn, +) { + if (!splitKeys || splitKeys.length === 0) { + return aggregateData(data, fieldsToSummarize, options, getValue); + } + + const [firstSplitKey, ...restSplitKeys] = splitKeys; + + const groupedData = data.reduce((acc, item) => { + let keyValuee = getValue(item, firstSplitKey) as string; + + if (typeof keyValuee === 'object') { + keyValuee = JSON.stringify(keyValuee); + } + + if (options.skipEmptySplitFields && typeof keyValuee !== 'number' && !keyValuee) { + return acc; + } + + if (acc[keyValuee] === undefined) { + acc[keyValuee] = [item]; + } else { + (acc[keyValuee] as IDataObject[]).push(item); + } + return acc; + }, {} as IDataObject); + + return Object.keys(groupedData).reduce((acc, key) => { + const value = groupedData[key] as IDataObject[]; + acc[key] = splitData(restSplitKeys, value, fieldsToSummarize, options, getValue); + return acc; + }, {} as IDataObject); +} + +function aggregationToArray( + aggregationResult: IDataObject, + fieldsToSplitBy: string[], + previousStage: IDataObject = {}, +) { + const returnData: IDataObject[] = []; + fieldsToSplitBy = parseFieldName(fieldsToSplitBy); + const splitFieldName = fieldsToSplitBy[0]; + const isNext = fieldsToSplitBy[1]; + + if (isNext === undefined) { + for (const fieldName of Object.keys(aggregationResult)) { + returnData.push({ + ...previousStage, + [splitFieldName]: fieldName, + ...(aggregationResult[fieldName] as IDataObject), + }); + } + return returnData; + } else { + for (const key of Object.keys(aggregationResult)) { + returnData.push( + ...aggregationToArray(aggregationResult[key] as IDataObject, fieldsToSplitBy.slice(1), { + ...previousStage, + [splitFieldName]: key, + }), + ); + } + return returnData; + } +} + +export async function execute( + this: IExecuteFunctions, + items: INodeExecutionData[], +): Promise { + const newItems = items.map(({ json }, i) => ({ ...json, _itemIndex: i })); + + const options = this.getNodeParameter('options', 0, {}) as SummarizeOptions; + + const fieldsToSplitBy = (this.getNodeParameter('fieldsToSplitBy', 0, '') as string) + .split(',') + .map((field) => field.trim()) + .filter((field) => field); + + const fieldsToSummarize = this.getNodeParameter( + 'fieldsToSummarize.values', + 0, + [], + ) as Aggregations; + + if (fieldsToSummarize.filter((aggregation) => aggregation.field !== '').length === 0) { + throw new NodeOperationError( + this.getNode(), + "You need to add at least one aggregation to 'Fields to Summarize' with non empty 'Field'", + ); + } + + const getValue = fieldValueGetter(options.disableDotNotation); + + const nodeVersion = this.getNode().typeVersion; + + if (nodeVersion < 2.1) { + checkIfFieldExists.call(this, newItems, fieldsToSummarize, getValue); + } + + const aggregationResult = splitData( + fieldsToSplitBy, + newItems, + fieldsToSummarize, + options, + getValue, + ); + + if (options.outputFormat === 'singleItem') { + const executionData: INodeExecutionData = { + json: aggregationResult, + pairedItem: newItems.map((_v, index) => ({ + item: index, + })), + }; + return [executionData]; + } else { + if (!fieldsToSplitBy.length) { + const { pairedItems, ...json } = aggregationResult; + const executionData: INodeExecutionData = { + json, + pairedItem: ((pairedItems as number[]) || []).map((index: number) => ({ + item: index, + })), + }; + return [executionData]; + } + const returnData = aggregationToArray(aggregationResult, fieldsToSplitBy); + const executionData = returnData.map((item) => { + const { pairedItems, ...json } = item; + return { + json, + pairedItem: ((pairedItems as number[]) || []).map((index: number) => ({ + item: index, + })), + }; + }); + return executionData; + } +} diff --git a/packages/nodes-base/package.json b/packages/nodes-base/package.json index 4a7ed3a3ac73b..f3921a5566f89 100644 --- a/packages/nodes-base/package.json +++ b/packages/nodes-base/package.json @@ -784,7 +784,13 @@ "dist/nodes/Zendesk/ZendeskTrigger.node.js", "dist/nodes/Zoho/ZohoCrm.node.js", "dist/nodes/Zoom/Zoom.node.js", - "dist/nodes/Zulip/Zulip.node.js" + "dist/nodes/Zulip/Zulip.node.js", + "dist/nodes/Transform/Aggregate/Aggregate.node.js", + "dist/nodes/Transform/Limit/Limit.node.js", + "dist/nodes/Transform/RemoveDuplicates/RemoveDuplicates.node.js", + "dist/nodes/Transform/SplitOut/SplitOut.node.js", + "dist/nodes/Transform/Sort/Sort.node.js", + "dist/nodes/Transform/Summarize/Summarize.node.js" ] }, "devDependencies": { From ce086d53e4c00206be8aacc85bd829f82b150814 Mon Sep 17 00:00:00 2001 From: Elias Meire Date: Mon, 20 Nov 2023 09:28:05 +0100 Subject: [PATCH 03/17] Split ItemsList node into 1 node per operation --- .../Transform/Aggregate/Aggregate.node.ts | 414 ++++++++++++++++++ .../Aggregate/aggregate.operation.ts | 406 ----------------- .../nodes/Transform/Aggregate/aggregate.svg | 13 + .../nodes/Transform/Aggregate/utils.ts | 60 +++ .../nodes/Transform/Limit/Limit.node.ts | 70 +++ .../nodes/Transform/Limit/limit.operation.ts | 61 --- .../nodes/Transform/Limit/limit.svg | 13 + .../RemoveDuplicates/RemoveDuplicates.node.ts | 262 +++++++++++ .../removeDuplicates.operation.ts | 244 ----------- .../RemoveDuplicates/removeDuplicates.svg | 13 + .../nodes/Transform/RemoveDuplicates/utils.ts | 36 ++ .../nodes/Transform/Sort/Sort.node.ts | 157 +++++++ .../nodes/Transform/Sort/sort.operation.ts | 275 ------------ .../nodes-base/nodes/Transform/Sort/sort.svg | 13 + .../nodes-base/nodes/Transform/Sort/utils.ts | 31 ++ .../nodes/Transform/SplitOut/SplitOut.node.ts | 259 +++++++++++ .../nodes/Transform/SplitOut/splitOut.svg | 13 + .../nodes-base/nodes/Transform/utils/utils.ts | 14 + 18 files changed, 1368 insertions(+), 986 deletions(-) delete mode 100644 packages/nodes-base/nodes/Transform/Aggregate/aggregate.operation.ts create mode 100644 packages/nodes-base/nodes/Transform/Aggregate/aggregate.svg create mode 100644 packages/nodes-base/nodes/Transform/Aggregate/utils.ts delete mode 100644 packages/nodes-base/nodes/Transform/Limit/limit.operation.ts create mode 100644 packages/nodes-base/nodes/Transform/Limit/limit.svg delete mode 100644 packages/nodes-base/nodes/Transform/RemoveDuplicates/removeDuplicates.operation.ts create mode 100644 packages/nodes-base/nodes/Transform/RemoveDuplicates/removeDuplicates.svg create mode 100644 packages/nodes-base/nodes/Transform/RemoveDuplicates/utils.ts delete mode 100644 packages/nodes-base/nodes/Transform/Sort/sort.operation.ts create mode 100644 packages/nodes-base/nodes/Transform/Sort/sort.svg create mode 100644 packages/nodes-base/nodes/Transform/Sort/utils.ts create mode 100644 packages/nodes-base/nodes/Transform/SplitOut/splitOut.svg create mode 100644 packages/nodes-base/nodes/Transform/utils/utils.ts diff --git a/packages/nodes-base/nodes/Transform/Aggregate/Aggregate.node.ts b/packages/nodes-base/nodes/Transform/Aggregate/Aggregate.node.ts index e69de29bb2d1d..fd918290a4b20 100644 --- a/packages/nodes-base/nodes/Transform/Aggregate/Aggregate.node.ts +++ b/packages/nodes-base/nodes/Transform/Aggregate/Aggregate.node.ts @@ -0,0 +1,414 @@ +import get from 'lodash/get'; +import isEmpty from 'lodash/isEmpty'; +import set from 'lodash/set'; +import { + NodeOperationError, + type IDataObject, + type IExecuteFunctions, + type INodeExecutionData, + type INodeType, + type INodeTypeDescription, + type IPairedItemData, +} from 'n8n-workflow'; +import { prepareFieldsArray } from '../utils/utils'; +import { addBinariesToItem } from './utils'; + +export class Aggregate implements INodeType { + description: INodeTypeDescription = { + displayName: 'Aggregate', + name: 'aggregate', + icon: 'file:aggregate.svg', + group: ['transform'], + subtitle: '', + version: 1, + description: 'Combine a field from many items into a list in a single item', + defaults: { + name: 'Aggregate', + }, + inputs: ['main'], + outputs: ['main'], + properties: [ + { + displayName: 'Aggregate', + name: 'aggregate', + type: 'options', + default: 'aggregateIndividualFields', + options: [ + { + name: 'Individual Fields', + value: 'aggregateIndividualFields', + }, + { + name: 'All Item Data (Into a Single List)', + value: 'aggregateAllItemData', + }, + ], + }, + { + displayName: 'Fields To Aggregate', + name: 'fieldsToAggregate', + type: 'fixedCollection', + typeOptions: { + multipleValues: true, + }, + placeholder: 'Add Field To Aggregate', + default: { fieldToAggregate: [{ fieldToAggregate: '', renameField: false }] }, + displayOptions: { + show: { + aggregate: ['aggregateIndividualFields'], + }, + }, + options: [ + { + displayName: '', + name: 'fieldToAggregate', + values: [ + { + displayName: 'Input Field Name', + name: 'fieldToAggregate', + type: 'string', + default: '', + description: 'The name of a field in the input items to aggregate together', + // eslint-disable-next-line n8n-nodes-base/node-param-placeholder-miscased-id + placeholder: 'e.g. id', + hint: ' Enter the field name as text', + requiresDataPath: 'single', + }, + { + displayName: 'Rename Field', + name: 'renameField', + type: 'boolean', + default: false, + description: 'Whether to give the field a different name in the output', + }, + { + displayName: 'Output Field Name', + name: 'outputFieldName', + displayOptions: { + show: { + renameField: [true], + }, + }, + type: 'string', + default: '', + description: + 'The name of the field to put the aggregated data in. Leave blank to use the input field name.', + requiresDataPath: 'single', + }, + ], + }, + ], + }, + { + displayName: 'Put Output in Field', + name: 'destinationFieldName', + type: 'string', + displayOptions: { + show: { + aggregate: ['aggregateAllItemData'], + }, + }, + default: 'data', + description: 'The name of the output field to put the data in', + }, + { + displayName: 'Include', + name: 'include', + type: 'options', + default: 'allFields', + options: [ + { + name: 'All Fields', + value: 'allFields', + }, + { + name: 'Specified Fields', + value: 'specifiedFields', + }, + { + name: 'All Fields Except', + value: 'allFieldsExcept', + }, + ], + displayOptions: { + show: { + aggregate: ['aggregateAllItemData'], + }, + }, + }, + { + displayName: 'Fields To Exclude', + name: 'fieldsToExclude', + type: 'string', + placeholder: 'e.g. email, name', + default: '', + requiresDataPath: 'multiple', + displayOptions: { + show: { + aggregate: ['aggregateAllItemData'], + include: ['allFieldsExcept'], + }, + }, + }, + { + displayName: 'Fields To Include', + name: 'fieldsToInclude', + type: 'string', + placeholder: 'e.g. email, name', + default: '', + requiresDataPath: 'multiple', + displayOptions: { + show: { + aggregate: ['aggregateAllItemData'], + include: ['specifiedFields'], + }, + }, + }, + { + displayName: 'Options', + name: 'options', + type: 'collection', + placeholder: 'Add Field', + default: {}, + options: [ + { + displayName: 'Disable Dot Notation', + name: 'disableDotNotation', + type: 'boolean', + default: false, + description: + 'Whether to disallow referencing child fields using `parent.child` in the field name', + displayOptions: { + hide: { + '/aggregate': ['aggregateAllItemData'], + }, + }, + }, + { + displayName: 'Merge Lists', + name: 'mergeLists', + type: 'boolean', + default: false, + description: + 'Whether to merge the output into a single flat list (rather than a list of lists), if the field to aggregate is a list', + displayOptions: { + hide: { + '/aggregate': ['aggregateAllItemData'], + }, + }, + }, + { + displayName: 'Include Binaries', + name: 'includeBinaries', + type: 'boolean', + default: false, + description: 'Whether to include the binary data in the new item', + }, + { + displayName: 'Keep Only Unique Binaries', + name: 'keepOnlyUnique', + type: 'boolean', + default: false, + description: + 'Whether to keep only unique binaries by comparing mime types, file types, file sizes and file extensions', + displayOptions: { + show: { + includeBinaries: [true], + }, + }, + }, + { + displayName: 'Keep Missing And Null Values', + name: 'keepMissing', + type: 'boolean', + default: false, + description: + 'Whether to add a null entry to the aggregated list when there is a missing or null value', + displayOptions: { + hide: { + '/aggregate': ['aggregateAllItemData'], + }, + }, + }, + ], + }, + ], + }; + + async execute(this: IExecuteFunctions): Promise { + let returnData: INodeExecutionData = { json: {}, pairedItem: [] }; + const items = this.getInputData(); + + const aggregate = this.getNodeParameter('aggregate', 0, '') as string; + + if (aggregate === 'aggregateIndividualFields') { + const disableDotNotation = this.getNodeParameter( + 'options.disableDotNotation', + 0, + false, + ) as boolean; + const mergeLists = this.getNodeParameter('options.mergeLists', 0, false) as boolean; + const fieldsToAggregate = this.getNodeParameter( + 'fieldsToAggregate.fieldToAggregate', + 0, + [], + ) as [{ fieldToAggregate: string; renameField: boolean; outputFieldName: string }]; + const keepMissing = this.getNodeParameter('options.keepMissing', 0, false) as boolean; + + if (!fieldsToAggregate.length) { + throw new NodeOperationError(this.getNode(), 'No fields specified', { + description: 'Please add a field to aggregate', + }); + } + + const newItem: INodeExecutionData = { + json: {}, + pairedItem: Array.from({ length: items.length }, (_, i) => i).map((index) => { + return { + item: index, + }; + }), + }; + + const values: { [key: string]: any } = {}; + const outputFields: string[] = []; + + for (const { fieldToAggregate, outputFieldName, renameField } of fieldsToAggregate) { + const field = renameField ? outputFieldName : fieldToAggregate; + + if (outputFields.includes(field)) { + throw new NodeOperationError( + this.getNode(), + `The '${field}' output field is used more than once`, + { description: 'Please make sure each output field name is unique' }, + ); + } else { + outputFields.push(field); + } + + const getFieldToAggregate = () => + !disableDotNotation && fieldToAggregate.includes('.') + ? fieldToAggregate.split('.').pop() + : fieldToAggregate; + + const _outputFieldName = outputFieldName + ? outputFieldName + : (getFieldToAggregate() as string); + + if (fieldToAggregate !== '') { + values[_outputFieldName] = []; + for (let i = 0; i < items.length; i++) { + if (!disableDotNotation) { + let value = get(items[i].json, fieldToAggregate); + + if (!keepMissing) { + if (Array.isArray(value)) { + value = value.filter((entry) => entry !== null); + } else if (value === null || value === undefined) { + continue; + } + } + + if (Array.isArray(value) && mergeLists) { + values[_outputFieldName].push(...value); + } else { + values[_outputFieldName].push(value); + } + } else { + let value = items[i].json[fieldToAggregate]; + + if (!keepMissing) { + if (Array.isArray(value)) { + value = value.filter((entry) => entry !== null); + } else if (value === null || value === undefined) { + continue; + } + } + + if (Array.isArray(value) && mergeLists) { + values[_outputFieldName].push(...value); + } else { + values[_outputFieldName].push(value); + } + } + } + } + } + + for (const key of Object.keys(values)) { + if (!disableDotNotation) { + set(newItem.json, key, values[key]); + } else { + newItem.json[key] = values[key]; + } + } + + returnData = newItem; + } else { + let newItems: IDataObject[] = items.map((item) => item.json); + let pairedItem: IPairedItemData[] = []; + const destinationFieldName = this.getNodeParameter('destinationFieldName', 0) as string; + + const fieldsToExclude = prepareFieldsArray( + this.getNodeParameter('fieldsToExclude', 0, '') as string, + 'Fields To Exclude', + ); + + const fieldsToInclude = prepareFieldsArray( + this.getNodeParameter('fieldsToInclude', 0, '') as string, + 'Fields To Include', + ); + + if (fieldsToExclude.length || fieldsToInclude.length) { + newItems = newItems.reduce((acc, item, index) => { + const newItem: IDataObject = {}; + let outputFields = Object.keys(item); + + if (fieldsToExclude.length) { + outputFields = outputFields.filter((key) => !fieldsToExclude.includes(key)); + } + if (fieldsToInclude.length) { + outputFields = outputFields.filter((key) => + fieldsToInclude.length ? fieldsToInclude.includes(key) : true, + ); + } + + outputFields.forEach((key) => { + newItem[key] = item[key]; + }); + + if (isEmpty(newItem)) { + return acc; + } + + pairedItem.push({ item: index }); + return acc.concat([newItem]); + }, [] as IDataObject[]); + } else { + pairedItem = Array.from({ length: newItems.length }, (_, item) => ({ + item, + })); + } + + const output: INodeExecutionData = { json: { [destinationFieldName]: newItems }, pairedItem }; + + returnData = output; + } + + const includeBinaries = this.getNodeParameter('options.includeBinaries', 0, false) as boolean; + + if (includeBinaries) { + const pairedItems = (returnData.pairedItem || []) as IPairedItemData[]; + + const aggregatedItems = pairedItems.map((item) => { + return items[item.item]; + }); + + const keepOnlyUnique = this.getNodeParameter('options.keepOnlyUnique', 0, false) as boolean; + + addBinariesToItem(returnData, aggregatedItems, keepOnlyUnique); + } + + return [[returnData]]; + } +} diff --git a/packages/nodes-base/nodes/Transform/Aggregate/aggregate.operation.ts b/packages/nodes-base/nodes/Transform/Aggregate/aggregate.operation.ts deleted file mode 100644 index 138fd2d2767b7..0000000000000 --- a/packages/nodes-base/nodes/Transform/Aggregate/aggregate.operation.ts +++ /dev/null @@ -1,406 +0,0 @@ -import type { - IDataObject, - IExecuteFunctions, - INodeExecutionData, - INodeProperties, - IPairedItemData, -} from 'n8n-workflow'; -import { NodeOperationError } from 'n8n-workflow'; - -import get from 'lodash/get'; -import isEmpty from 'lodash/isEmpty'; -import set from 'lodash/set'; - -import { addBinariesToItem, prepareFieldsArray } from '../../helpers/utils'; -import { disableDotNotationBoolean } from '../common.descriptions'; -import { updateDisplayOptions } from '@utils/utilities'; - -const properties: INodeProperties[] = [ - { - displayName: 'Aggregate', - name: 'aggregate', - type: 'options', - default: 'aggregateIndividualFields', - options: [ - { - name: 'Individual Fields', - value: 'aggregateIndividualFields', - }, - { - name: 'All Item Data (Into a Single List)', - value: 'aggregateAllItemData', - }, - ], - }, - { - displayName: 'Fields To Aggregate', - name: 'fieldsToAggregate', - type: 'fixedCollection', - typeOptions: { - multipleValues: true, - }, - placeholder: 'Add Field To Aggregate', - default: { fieldToAggregate: [{ fieldToAggregate: '', renameField: false }] }, - displayOptions: { - show: { - aggregate: ['aggregateIndividualFields'], - }, - }, - options: [ - { - displayName: '', - name: 'fieldToAggregate', - values: [ - { - displayName: 'Input Field Name', - name: 'fieldToAggregate', - type: 'string', - default: '', - description: 'The name of a field in the input items to aggregate together', - // eslint-disable-next-line n8n-nodes-base/node-param-placeholder-miscased-id - placeholder: 'e.g. id', - hint: ' Enter the field name as text', - requiresDataPath: 'single', - }, - { - displayName: 'Rename Field', - name: 'renameField', - type: 'boolean', - default: false, - description: 'Whether to give the field a different name in the output', - }, - { - displayName: 'Output Field Name', - name: 'outputFieldName', - displayOptions: { - show: { - renameField: [true], - }, - }, - type: 'string', - default: '', - description: - 'The name of the field to put the aggregated data in. Leave blank to use the input field name.', - requiresDataPath: 'single', - }, - ], - }, - ], - }, - { - displayName: 'Put Output in Field', - name: 'destinationFieldName', - type: 'string', - displayOptions: { - show: { - aggregate: ['aggregateAllItemData'], - }, - }, - default: 'data', - description: 'The name of the output field to put the data in', - }, - { - displayName: 'Include', - name: 'include', - type: 'options', - default: 'allFields', - options: [ - { - name: 'All Fields', - value: 'allFields', - }, - { - name: 'Specified Fields', - value: 'specifiedFields', - }, - { - name: 'All Fields Except', - value: 'allFieldsExcept', - }, - ], - displayOptions: { - show: { - aggregate: ['aggregateAllItemData'], - }, - }, - }, - { - displayName: 'Fields To Exclude', - name: 'fieldsToExclude', - type: 'string', - placeholder: 'e.g. email, name', - default: '', - requiresDataPath: 'multiple', - displayOptions: { - show: { - aggregate: ['aggregateAllItemData'], - include: ['allFieldsExcept'], - }, - }, - }, - { - displayName: 'Fields To Include', - name: 'fieldsToInclude', - type: 'string', - placeholder: 'e.g. email, name', - default: '', - requiresDataPath: 'multiple', - displayOptions: { - show: { - aggregate: ['aggregateAllItemData'], - include: ['specifiedFields'], - }, - }, - }, - { - displayName: 'Options', - name: 'options', - type: 'collection', - placeholder: 'Add Field', - default: {}, - options: [ - { - ...disableDotNotationBoolean, - displayOptions: { - hide: { - '/aggregate': ['aggregateAllItemData'], - }, - }, - }, - { - displayName: 'Merge Lists', - name: 'mergeLists', - type: 'boolean', - default: false, - description: - 'Whether to merge the output into a single flat list (rather than a list of lists), if the field to aggregate is a list', - displayOptions: { - hide: { - '/aggregate': ['aggregateAllItemData'], - }, - }, - }, - { - displayName: 'Include Binaries', - name: 'includeBinaries', - type: 'boolean', - default: false, - description: 'Whether to include the binary data in the new item', - }, - { - displayName: 'Keep Only Unique Binaries', - name: 'keepOnlyUnique', - type: 'boolean', - default: false, - description: - 'Whether to keep only unique binaries by comparing mime types, file types, file sizes and file extensions', - displayOptions: { - show: { - includeBinaries: [true], - }, - }, - }, - { - displayName: 'Keep Missing And Null Values', - name: 'keepMissing', - type: 'boolean', - default: false, - description: - 'Whether to add a null entry to the aggregated list when there is a missing or null value', - displayOptions: { - hide: { - '/aggregate': ['aggregateAllItemData'], - }, - }, - }, - ], - }, -]; - -const displayOptions = { - show: { - resource: ['itemList'], - operation: ['concatenateItems'], - }, -}; - -export const description = updateDisplayOptions(displayOptions, properties); - -export async function execute( - this: IExecuteFunctions, - items: INodeExecutionData[], -): Promise { - let returnData: INodeExecutionData = { json: {}, pairedItem: [] }; - - const aggregate = this.getNodeParameter('aggregate', 0, '') as string; - - if (aggregate === 'aggregateIndividualFields') { - const disableDotNotation = this.getNodeParameter( - 'options.disableDotNotation', - 0, - false, - ) as boolean; - const mergeLists = this.getNodeParameter('options.mergeLists', 0, false) as boolean; - const fieldsToAggregate = this.getNodeParameter( - 'fieldsToAggregate.fieldToAggregate', - 0, - [], - ) as [{ fieldToAggregate: string; renameField: boolean; outputFieldName: string }]; - const keepMissing = this.getNodeParameter('options.keepMissing', 0, false) as boolean; - - if (!fieldsToAggregate.length) { - throw new NodeOperationError(this.getNode(), 'No fields specified', { - description: 'Please add a field to aggregate', - }); - } - - const newItem: INodeExecutionData = { - json: {}, - pairedItem: Array.from({ length: items.length }, (_, i) => i).map((index) => { - return { - item: index, - }; - }), - }; - - const values: { [key: string]: any } = {}; - const outputFields: string[] = []; - - for (const { fieldToAggregate, outputFieldName, renameField } of fieldsToAggregate) { - const field = renameField ? outputFieldName : fieldToAggregate; - - if (outputFields.includes(field)) { - throw new NodeOperationError( - this.getNode(), - `The '${field}' output field is used more than once`, - { description: 'Please make sure each output field name is unique' }, - ); - } else { - outputFields.push(field); - } - - const getFieldToAggregate = () => - !disableDotNotation && fieldToAggregate.includes('.') - ? fieldToAggregate.split('.').pop() - : fieldToAggregate; - - const _outputFieldName = outputFieldName - ? outputFieldName - : (getFieldToAggregate() as string); - - if (fieldToAggregate !== '') { - values[_outputFieldName] = []; - for (let i = 0; i < items.length; i++) { - if (!disableDotNotation) { - let value = get(items[i].json, fieldToAggregate); - - if (!keepMissing) { - if (Array.isArray(value)) { - value = value.filter((entry) => entry !== null); - } else if (value === null || value === undefined) { - continue; - } - } - - if (Array.isArray(value) && mergeLists) { - values[_outputFieldName].push(...value); - } else { - values[_outputFieldName].push(value); - } - } else { - let value = items[i].json[fieldToAggregate]; - - if (!keepMissing) { - if (Array.isArray(value)) { - value = value.filter((entry) => entry !== null); - } else if (value === null || value === undefined) { - continue; - } - } - - if (Array.isArray(value) && mergeLists) { - values[_outputFieldName].push(...value); - } else { - values[_outputFieldName].push(value); - } - } - } - } - } - - for (const key of Object.keys(values)) { - if (!disableDotNotation) { - set(newItem.json, key, values[key]); - } else { - newItem.json[key] = values[key]; - } - } - - returnData = newItem; - } else { - let newItems: IDataObject[] = items.map((item) => item.json); - let pairedItem: IPairedItemData[] = []; - const destinationFieldName = this.getNodeParameter('destinationFieldName', 0) as string; - - const fieldsToExclude = prepareFieldsArray( - this.getNodeParameter('fieldsToExclude', 0, '') as string, - 'Fields To Exclude', - ); - - const fieldsToInclude = prepareFieldsArray( - this.getNodeParameter('fieldsToInclude', 0, '') as string, - 'Fields To Include', - ); - - if (fieldsToExclude.length || fieldsToInclude.length) { - newItems = newItems.reduce((acc, item, index) => { - const newItem: IDataObject = {}; - let outputFields = Object.keys(item); - - if (fieldsToExclude.length) { - outputFields = outputFields.filter((key) => !fieldsToExclude.includes(key)); - } - if (fieldsToInclude.length) { - outputFields = outputFields.filter((key) => - fieldsToInclude.length ? fieldsToInclude.includes(key) : true, - ); - } - - outputFields.forEach((key) => { - newItem[key] = item[key]; - }); - - if (isEmpty(newItem)) { - return acc; - } - - pairedItem.push({ item: index }); - return acc.concat([newItem]); - }, [] as IDataObject[]); - } else { - pairedItem = Array.from({ length: newItems.length }, (_, item) => ({ - item, - })); - } - - const output: INodeExecutionData = { json: { [destinationFieldName]: newItems }, pairedItem }; - - returnData = output; - } - - const includeBinaries = this.getNodeParameter('options.includeBinaries', 0, false) as boolean; - - if (includeBinaries) { - const pairedItems = (returnData.pairedItem || []) as IPairedItemData[]; - - const aggregatedItems = pairedItems.map((item) => { - return items[item.item]; - }); - - const keepOnlyUnique = this.getNodeParameter('options.keepOnlyUnique', 0, false) as boolean; - - addBinariesToItem(returnData, aggregatedItems, keepOnlyUnique); - } - - return [returnData]; -} diff --git a/packages/nodes-base/nodes/Transform/Aggregate/aggregate.svg b/packages/nodes-base/nodes/Transform/Aggregate/aggregate.svg new file mode 100644 index 0000000000000..56100defcee2d --- /dev/null +++ b/packages/nodes-base/nodes/Transform/Aggregate/aggregate.svg @@ -0,0 +1,13 @@ + + + + + + \ No newline at end of file diff --git a/packages/nodes-base/nodes/Transform/Aggregate/utils.ts b/packages/nodes-base/nodes/Transform/Aggregate/utils.ts new file mode 100644 index 0000000000000..5a4f1e10fd4be --- /dev/null +++ b/packages/nodes-base/nodes/Transform/Aggregate/utils.ts @@ -0,0 +1,60 @@ +import type { IBinaryData, INodeExecutionData } from 'n8n-workflow'; + +type PartialBinaryData = Omit; +const isBinaryUniqueSetup = () => { + const binaries: PartialBinaryData[] = []; + return (binary: IBinaryData) => { + for (const existingBinary of binaries) { + if ( + existingBinary.mimeType === binary.mimeType && + existingBinary.fileType === binary.fileType && + existingBinary.fileSize === binary.fileSize && + existingBinary.fileExtension === binary.fileExtension + ) { + return false; + } + } + + binaries.push({ + mimeType: binary.mimeType, + fileType: binary.fileType, + fileSize: binary.fileSize, + fileExtension: binary.fileExtension, + }); + + return true; + }; +}; + +export function addBinariesToItem( + newItem: INodeExecutionData, + items: INodeExecutionData[], + uniqueOnly?: boolean, +) { + const isBinaryUnique = uniqueOnly ? isBinaryUniqueSetup() : undefined; + + for (const item of items) { + if (item.binary === undefined) continue; + + for (const key of Object.keys(item.binary)) { + if (!newItem.binary) newItem.binary = {}; + let binaryKey = key; + const binary = item.binary[key]; + + if (isBinaryUnique && !isBinaryUnique(binary)) { + continue; + } + + // If the binary key already exists add a suffix to it + let i = 1; + while (newItem.binary[binaryKey] !== undefined) { + binaryKey = `${key}_${i}`; + i++; + } + + newItem.binary[binaryKey] = binary; + } + } + + return newItem; +} diff --git a/packages/nodes-base/nodes/Transform/Limit/Limit.node.ts b/packages/nodes-base/nodes/Transform/Limit/Limit.node.ts index e69de29bb2d1d..0415520801aad 100644 --- a/packages/nodes-base/nodes/Transform/Limit/Limit.node.ts +++ b/packages/nodes-base/nodes/Transform/Limit/Limit.node.ts @@ -0,0 +1,70 @@ +import type { + IExecuteFunctions, + INodeExecutionData, + INodeType, + INodeTypeDescription, +} from 'n8n-workflow'; + +export class Limit implements INodeType { + description: INodeTypeDescription = { + displayName: 'Limit', + name: 'limit', + icon: 'file:limit.svg', + group: ['transform'], + subtitle: '', + version: 1, + description: 'Restrict the number of items', + defaults: { + name: 'Limit', + }, + inputs: ['main'], + outputs: ['main'], + properties: [ + { + displayName: 'Max Items', + name: 'maxItems', + type: 'number', + typeOptions: { + minValue: 1, + }, + default: 1, + description: 'If there are more items than this number, some are removed', + }, + { + displayName: 'Keep', + name: 'keep', + type: 'options', + options: [ + { + name: 'First Items', + value: 'firstItems', + }, + { + name: 'Last Items', + value: 'lastItems', + }, + ], + default: 'firstItems', + description: 'When removing items, whether to keep the ones at the start or the ending', + }, + ], + }; + + async execute(this: IExecuteFunctions): Promise { + const items = this.getInputData(); + let returnData = items; + const maxItems = this.getNodeParameter('maxItems', 0) as number; + const keep = this.getNodeParameter('keep', 0) as string; + + if (maxItems > items.length) { + return [returnData]; + } + + if (keep === 'firstItems') { + returnData = items.slice(0, maxItems); + } else { + returnData = items.slice(items.length - maxItems, items.length); + } + return [returnData]; + } +} diff --git a/packages/nodes-base/nodes/Transform/Limit/limit.operation.ts b/packages/nodes-base/nodes/Transform/Limit/limit.operation.ts deleted file mode 100644 index 5671cf246fa03..0000000000000 --- a/packages/nodes-base/nodes/Transform/Limit/limit.operation.ts +++ /dev/null @@ -1,61 +0,0 @@ -import type { IExecuteFunctions, INodeExecutionData, INodeProperties } from 'n8n-workflow'; -import { updateDisplayOptions } from '@utils/utilities'; - -const properties: INodeProperties[] = [ - { - displayName: 'Max Items', - name: 'maxItems', - type: 'number', - typeOptions: { - minValue: 1, - }, - default: 1, - description: 'If there are more items than this number, some are removed', - }, - { - displayName: 'Keep', - name: 'keep', - type: 'options', - options: [ - { - name: 'First Items', - value: 'firstItems', - }, - { - name: 'Last Items', - value: 'lastItems', - }, - ], - default: 'firstItems', - description: 'When removing items, whether to keep the ones at the start or the ending', - }, -]; - -const displayOptions = { - show: { - resource: ['itemList'], - operation: ['limit'], - }, -}; - -export const description = updateDisplayOptions(displayOptions, properties); - -export async function execute( - this: IExecuteFunctions, - items: INodeExecutionData[], -): Promise { - let returnData = items; - const maxItems = this.getNodeParameter('maxItems', 0) as number; - const keep = this.getNodeParameter('keep', 0) as string; - - if (maxItems > items.length) { - return returnData; - } - - if (keep === 'firstItems') { - returnData = items.slice(0, maxItems); - } else { - returnData = items.slice(items.length - maxItems, items.length); - } - return returnData; -} diff --git a/packages/nodes-base/nodes/Transform/Limit/limit.svg b/packages/nodes-base/nodes/Transform/Limit/limit.svg new file mode 100644 index 0000000000000..56100defcee2d --- /dev/null +++ b/packages/nodes-base/nodes/Transform/Limit/limit.svg @@ -0,0 +1,13 @@ + + + + + + \ No newline at end of file diff --git a/packages/nodes-base/nodes/Transform/RemoveDuplicates/RemoveDuplicates.node.ts b/packages/nodes-base/nodes/Transform/RemoveDuplicates/RemoveDuplicates.node.ts index e69de29bb2d1d..79b7b80f8aad2 100644 --- a/packages/nodes-base/nodes/Transform/RemoveDuplicates/RemoveDuplicates.node.ts +++ b/packages/nodes-base/nodes/Transform/RemoveDuplicates/RemoveDuplicates.node.ts @@ -0,0 +1,262 @@ +import get from 'lodash/get'; +import isEqual from 'lodash/isEqual'; +import lt from 'lodash/lt'; +import pick from 'lodash/pick'; +import { + NodeOperationError, + type IExecuteFunctions, + type INodeExecutionData, + type INodeType, + type INodeTypeDescription, +} from 'n8n-workflow'; +import { prepareFieldsArray } from '../utils/utils'; +import { compareItems, flattenKeys } from './utils'; + +export class RemoveDuplicates implements INodeType { + description: INodeTypeDescription = { + displayName: 'Remove Duplicates', + name: 'removeDuplicates', + icon: 'file:removeDuplicates.svg', + group: ['transform'], + subtitle: '', + version: 1, + description: 'Remove extra items that are similar', + defaults: { + name: 'Remove Duplicates', + }, + inputs: ['main'], + outputs: ['main'], + properties: [ + { + displayName: 'Compare', + name: 'compare', + type: 'options', + options: [ + { + name: 'All Fields', + value: 'allFields', + }, + { + name: 'All Fields Except', + value: 'allFieldsExcept', + }, + { + name: 'Selected Fields', + value: 'selectedFields', + }, + ], + default: 'allFields', + description: 'The fields of the input items to compare to see if they are the same', + }, + { + displayName: 'Fields To Exclude', + name: 'fieldsToExclude', + type: 'string', + placeholder: 'e.g. email, name', + requiresDataPath: 'multiple', + description: 'Fields in the input to exclude from the comparison', + default: '', + displayOptions: { + show: { + compare: ['allFieldsExcept'], + }, + }, + }, + { + displayName: 'Fields To Compare', + name: 'fieldsToCompare', + type: 'string', + placeholder: 'e.g. email, name', + requiresDataPath: 'multiple', + description: 'Fields in the input to add to the comparison', + default: '', + displayOptions: { + show: { + compare: ['selectedFields'], + }, + }, + }, + { + displayName: 'Options', + name: 'options', + type: 'collection', + placeholder: 'Add Field', + default: {}, + displayOptions: { + show: { + compare: ['allFieldsExcept', 'selectedFields'], + }, + }, + options: [ + { + displayName: 'Disable Dot Notation', + name: 'disableDotNotation', + type: 'boolean', + default: false, + description: + 'Whether to disallow referencing child fields using `parent.child` in the field name', + }, + { + displayName: 'Remove Other Fields', + name: 'removeOtherFields', + type: 'boolean', + default: false, + description: + 'Whether to remove any fields that are not being compared. If disabled, will keep the values from the first of the duplicates.', + }, + ], + }, + ], + }; + + async execute(this: IExecuteFunctions): Promise { + const items = this.getInputData(); + const compare = this.getNodeParameter('compare', 0) as string; + const disableDotNotation = this.getNodeParameter( + 'options.disableDotNotation', + 0, + false, + ) as boolean; + const removeOtherFields = this.getNodeParameter( + 'options.removeOtherFields', + 0, + false, + ) as boolean; + + let keys = disableDotNotation + ? Object.keys(items[0].json) + : Object.keys(flattenKeys(items[0].json)); + + for (const item of items) { + for (const key of disableDotNotation + ? Object.keys(item.json) + : Object.keys(flattenKeys(item.json))) { + if (!keys.includes(key)) { + keys.push(key); + } + } + } + + if (compare === 'allFieldsExcept') { + const fieldsToExclude = prepareFieldsArray( + this.getNodeParameter('fieldsToExclude', 0, '') as string, + 'Fields To Exclude', + ); + + if (!fieldsToExclude.length) { + throw new NodeOperationError( + this.getNode(), + 'No fields specified. Please add a field to exclude from comparison', + ); + } + if (!disableDotNotation) { + keys = Object.keys(flattenKeys(items[0].json)); + } + keys = keys.filter((key) => !fieldsToExclude.includes(key)); + } + if (compare === 'selectedFields') { + const fieldsToCompare = prepareFieldsArray( + this.getNodeParameter('fieldsToCompare', 0, '') as string, + 'Fields To Compare', + ); + if (!fieldsToCompare.length) { + throw new NodeOperationError( + this.getNode(), + 'No fields specified. Please add a field to compare on', + ); + } + if (!disableDotNotation) { + keys = Object.keys(flattenKeys(items[0].json)); + } + keys = fieldsToCompare.map((key) => key.trim()); + } + + // This solution is O(nlogn) + // add original index to the items + const newItems = items.map( + (item, index) => + ({ + json: { ...item.json, __INDEX: index }, + pairedItem: { item: index }, + }) as INodeExecutionData, + ); + //sort items using the compare keys + newItems.sort((a, b) => { + let result = 0; + + for (const key of keys) { + let equal; + if (!disableDotNotation) { + equal = isEqual(get(a.json, key), get(b.json, key)); + } else { + equal = isEqual(a.json[key], b.json[key]); + } + if (!equal) { + let lessThan; + if (!disableDotNotation) { + lessThan = lt(get(a.json, key), get(b.json, key)); + } else { + lessThan = lt(a.json[key], b.json[key]); + } + result = lessThan ? -1 : 1; + break; + } + } + return result; + }); + + for (const key of keys) { + let type: any = undefined; + for (const item of newItems) { + if (key === '') { + throw new NodeOperationError(this.getNode(), 'Name of field to compare is blank'); + } + const value = !disableDotNotation ? get(item.json, key) : item.json[key]; + if (value === undefined && disableDotNotation && key.includes('.')) { + throw new NodeOperationError( + this.getNode(), + `'${key}' field is missing from some input items`, + { + description: + "If you're trying to use a nested field, make sure you turn off 'disable dot notation' in the node options", + }, + ); + } else if (value === undefined) { + throw new NodeOperationError( + this.getNode(), + `'${key}' field is missing from some input items`, + ); + } + if (type !== undefined && value !== undefined && type !== typeof value) { + throw new NodeOperationError(this.getNode(), `'${key}' isn't always the same type`, { + description: 'The type of this field varies between items', + }); + } else { + type = typeof value; + } + } + } + + // collect the original indexes of items to be removed + const removedIndexes: number[] = []; + let temp = newItems[0]; + for (let index = 1; index < newItems.length; index++) { + if (compareItems(newItems[index], temp, keys, disableDotNotation, this.getNode())) { + removedIndexes.push(newItems[index].json.__INDEX as unknown as number); + } else { + temp = newItems[index]; + } + } + + let returnData = items.filter((_, index) => !removedIndexes.includes(index)); + + if (removeOtherFields) { + returnData = returnData.map((item, index) => ({ + json: pick(item.json, ...keys), + pairedItem: { item: index }, + })); + } + + return [returnData]; + } +} diff --git a/packages/nodes-base/nodes/Transform/RemoveDuplicates/removeDuplicates.operation.ts b/packages/nodes-base/nodes/Transform/RemoveDuplicates/removeDuplicates.operation.ts deleted file mode 100644 index e97be1359ae79..0000000000000 --- a/packages/nodes-base/nodes/Transform/RemoveDuplicates/removeDuplicates.operation.ts +++ /dev/null @@ -1,244 +0,0 @@ -import type { IExecuteFunctions, INodeExecutionData, INodeProperties } from 'n8n-workflow'; -import { NodeOperationError } from 'n8n-workflow'; - -import get from 'lodash/get'; -import isEqual from 'lodash/isEqual'; -import lt from 'lodash/lt'; -import pick from 'lodash/pick'; - -import { compareItems, flattenKeys, prepareFieldsArray } from '../../helpers/utils'; -import { disableDotNotationBoolean } from '../common.descriptions'; -import { updateDisplayOptions } from '@utils/utilities'; - -const properties: INodeProperties[] = [ - { - displayName: 'Compare', - name: 'compare', - type: 'options', - options: [ - { - name: 'All Fields', - value: 'allFields', - }, - { - name: 'All Fields Except', - value: 'allFieldsExcept', - }, - { - name: 'Selected Fields', - value: 'selectedFields', - }, - ], - default: 'allFields', - description: 'The fields of the input items to compare to see if they are the same', - }, - { - displayName: 'Fields To Exclude', - name: 'fieldsToExclude', - type: 'string', - placeholder: 'e.g. email, name', - requiresDataPath: 'multiple', - description: 'Fields in the input to exclude from the comparison', - default: '', - displayOptions: { - show: { - compare: ['allFieldsExcept'], - }, - }, - }, - { - displayName: 'Fields To Compare', - name: 'fieldsToCompare', - type: 'string', - placeholder: 'e.g. email, name', - requiresDataPath: 'multiple', - description: 'Fields in the input to add to the comparison', - default: '', - displayOptions: { - show: { - compare: ['selectedFields'], - }, - }, - }, - { - displayName: 'Options', - name: 'options', - type: 'collection', - placeholder: 'Add Field', - default: {}, - displayOptions: { - show: { - compare: ['allFieldsExcept', 'selectedFields'], - }, - }, - options: [ - disableDotNotationBoolean, - { - displayName: 'Remove Other Fields', - name: 'removeOtherFields', - type: 'boolean', - default: false, - description: - 'Whether to remove any fields that are not being compared. If disabled, will keep the values from the first of the duplicates.', - }, - ], - }, -]; - -const displayOptions = { - show: { - resource: ['itemList'], - operation: ['removeDuplicates'], - }, -}; - -export const description = updateDisplayOptions(displayOptions, properties); - -export async function execute( - this: IExecuteFunctions, - items: INodeExecutionData[], -): Promise { - const compare = this.getNodeParameter('compare', 0) as string; - const disableDotNotation = this.getNodeParameter( - 'options.disableDotNotation', - 0, - false, - ) as boolean; - const removeOtherFields = this.getNodeParameter('options.removeOtherFields', 0, false) as boolean; - - let keys = disableDotNotation - ? Object.keys(items[0].json) - : Object.keys(flattenKeys(items[0].json)); - - for (const item of items) { - for (const key of disableDotNotation - ? Object.keys(item.json) - : Object.keys(flattenKeys(item.json))) { - if (!keys.includes(key)) { - keys.push(key); - } - } - } - - if (compare === 'allFieldsExcept') { - const fieldsToExclude = prepareFieldsArray( - this.getNodeParameter('fieldsToExclude', 0, '') as string, - 'Fields To Exclude', - ); - - if (!fieldsToExclude.length) { - throw new NodeOperationError( - this.getNode(), - 'No fields specified. Please add a field to exclude from comparison', - ); - } - if (!disableDotNotation) { - keys = Object.keys(flattenKeys(items[0].json)); - } - keys = keys.filter((key) => !fieldsToExclude.includes(key)); - } - if (compare === 'selectedFields') { - const fieldsToCompare = prepareFieldsArray( - this.getNodeParameter('fieldsToCompare', 0, '') as string, - 'Fields To Compare', - ); - if (!fieldsToCompare.length) { - throw new NodeOperationError( - this.getNode(), - 'No fields specified. Please add a field to compare on', - ); - } - if (!disableDotNotation) { - keys = Object.keys(flattenKeys(items[0].json)); - } - keys = fieldsToCompare.map((key) => key.trim()); - } - - // This solution is O(nlogn) - // add original index to the items - const newItems = items.map( - (item, index) => - ({ - json: { ...item.json, __INDEX: index }, - pairedItem: { item: index }, - }) as INodeExecutionData, - ); - //sort items using the compare keys - newItems.sort((a, b) => { - let result = 0; - - for (const key of keys) { - let equal; - if (!disableDotNotation) { - equal = isEqual(get(a.json, key), get(b.json, key)); - } else { - equal = isEqual(a.json[key], b.json[key]); - } - if (!equal) { - let lessThan; - if (!disableDotNotation) { - lessThan = lt(get(a.json, key), get(b.json, key)); - } else { - lessThan = lt(a.json[key], b.json[key]); - } - result = lessThan ? -1 : 1; - break; - } - } - return result; - }); - - for (const key of keys) { - let type: any = undefined; - for (const item of newItems) { - if (key === '') { - throw new NodeOperationError(this.getNode(), 'Name of field to compare is blank'); - } - const value = !disableDotNotation ? get(item.json, key) : item.json[key]; - if (value === undefined && disableDotNotation && key.includes('.')) { - throw new NodeOperationError( - this.getNode(), - `'${key}' field is missing from some input items`, - { - description: - "If you're trying to use a nested field, make sure you turn off 'disable dot notation' in the node options", - }, - ); - } else if (value === undefined) { - throw new NodeOperationError( - this.getNode(), - `'${key}' field is missing from some input items`, - ); - } - if (type !== undefined && value !== undefined && type !== typeof value) { - throw new NodeOperationError(this.getNode(), `'${key}' isn't always the same type`, { - description: 'The type of this field varies between items', - }); - } else { - type = typeof value; - } - } - } - - // collect the original indexes of items to be removed - const removedIndexes: number[] = []; - let temp = newItems[0]; - for (let index = 1; index < newItems.length; index++) { - if (compareItems(newItems[index], temp, keys, disableDotNotation, this.getNode())) { - removedIndexes.push(newItems[index].json.__INDEX as unknown as number); - } else { - temp = newItems[index]; - } - } - - let returnData = items.filter((_, index) => !removedIndexes.includes(index)); - - if (removeOtherFields) { - returnData = returnData.map((item, index) => ({ - json: pick(item.json, ...keys), - pairedItem: { item: index }, - })); - } - - return returnData; -} diff --git a/packages/nodes-base/nodes/Transform/RemoveDuplicates/removeDuplicates.svg b/packages/nodes-base/nodes/Transform/RemoveDuplicates/removeDuplicates.svg new file mode 100644 index 0000000000000..de349daca71ab --- /dev/null +++ b/packages/nodes-base/nodes/Transform/RemoveDuplicates/removeDuplicates.svg @@ -0,0 +1,13 @@ + + + + + + diff --git a/packages/nodes-base/nodes/Transform/RemoveDuplicates/utils.ts b/packages/nodes-base/nodes/Transform/RemoveDuplicates/utils.ts new file mode 100644 index 0000000000000..ff5d794092dbd --- /dev/null +++ b/packages/nodes-base/nodes/Transform/RemoveDuplicates/utils.ts @@ -0,0 +1,36 @@ +import get from 'lodash/get'; +import isEqual from 'lodash/isEqual'; +import isObject from 'lodash/isObject'; +import merge from 'lodash/merge'; +import reduce from 'lodash/reduce'; +import type { IDataObject, INode, INodeExecutionData } from 'n8n-workflow'; + +export const compareItems = ( + obj: INodeExecutionData, + obj2: INodeExecutionData, + keys: string[], + disableDotNotation: boolean, + _node: INode, +) => { + let result = true; + for (const key of keys) { + if (!disableDotNotation) { + if (!isEqual(get(obj.json, key), get(obj2.json, key))) { + result = false; + break; + } + } else { + if (!isEqual(obj.json[key], obj2.json[key])) { + result = false; + break; + } + } + } + return result; +}; + +export const flattenKeys = (obj: IDataObject, path: string[] = []): IDataObject => { + return !isObject(obj) + ? { [path.join('.')]: obj } + : reduce(obj, (cum, next, key) => merge(cum, flattenKeys(next as IDataObject, [...path, key])), {}); //prettier-ignore +}; diff --git a/packages/nodes-base/nodes/Transform/Sort/Sort.node.ts b/packages/nodes-base/nodes/Transform/Sort/Sort.node.ts index e69de29bb2d1d..67f131eebfe9b 100644 --- a/packages/nodes-base/nodes/Transform/Sort/Sort.node.ts +++ b/packages/nodes-base/nodes/Transform/Sort/Sort.node.ts @@ -0,0 +1,157 @@ +import get from 'lodash/get'; +import isEqual from 'lodash/isEqual'; +import lt from 'lodash/lt'; +import { + NodeOperationError, + type IDataObject, + type IExecuteFunctions, + type INodeExecutionData, + type INodeType, + type INodeTypeDescription, +} from 'n8n-workflow'; +import { shuffleArray, sortByCode } from './utils'; + +export class Sort implements INodeType { + description: INodeTypeDescription = { + displayName: 'Sort', + name: 'sort', + icon: 'file:sort.svg', + group: ['transform'], + subtitle: '', + version: 1, + description: 'Change items order', + defaults: { + name: 'Sort', + }, + inputs: ['main'], + outputs: ['main'], + properties: [], + }; + + async execute(this: IExecuteFunctions): Promise { + const items = this.getInputData(); + let returnData = [...items]; + const type = this.getNodeParameter('type', 0) as string; + const disableDotNotation = this.getNodeParameter( + 'options.disableDotNotation', + 0, + false, + ) as boolean; + + if (type === 'random') { + shuffleArray(returnData); + return [returnData]; + } + + if (type === 'simple') { + const sortFieldsUi = this.getNodeParameter('sortFieldsUi', 0) as IDataObject; + const sortFields = sortFieldsUi.sortField as Array<{ + fieldName: string; + order: 'ascending' | 'descending'; + }>; + + if (!sortFields?.length) { + throw new NodeOperationError( + this.getNode(), + 'No sorting specified. Please add a field to sort by', + ); + } + + for (const { fieldName } of sortFields) { + let found = false; + for (const item of items) { + if (!disableDotNotation) { + if (get(item.json, fieldName) !== undefined) { + found = true; + } + } else if (item.json.hasOwnProperty(fieldName)) { + found = true; + } + } + if (!found && disableDotNotation && fieldName.includes('.')) { + throw new NodeOperationError( + this.getNode(), + `Couldn't find the field '${fieldName}' in the input data`, + { + description: + "If you're trying to use a nested field, make sure you turn off 'disable dot notation' in the node options", + }, + ); + } else if (!found) { + throw new NodeOperationError( + this.getNode(), + `Couldn't find the field '${fieldName}' in the input data`, + ); + } + } + + const sortFieldsWithDirection = sortFields.map((field) => ({ + name: field.fieldName, + dir: field.order === 'ascending' ? 1 : -1, + })); + + returnData.sort((a, b) => { + let result = 0; + for (const field of sortFieldsWithDirection) { + let equal; + if (!disableDotNotation) { + const _a = + typeof get(a.json, field.name) === 'string' + ? (get(a.json, field.name) as string).toLowerCase() + : get(a.json, field.name); + const _b = + typeof get(b.json, field.name) === 'string' + ? (get(b.json, field.name) as string).toLowerCase() + : get(b.json, field.name); + equal = isEqual(_a, _b); + } else { + const _a = + typeof a.json[field.name] === 'string' + ? (a.json[field.name] as string).toLowerCase() + : a.json[field.name]; + const _b = + typeof b.json[field.name] === 'string' + ? (b.json[field.name] as string).toLowerCase() + : b.json[field.name]; + equal = isEqual(_a, _b); + } + + if (!equal) { + let lessThan; + if (!disableDotNotation) { + const _a = + typeof get(a.json, field.name) === 'string' + ? (get(a.json, field.name) as string).toLowerCase() + : get(a.json, field.name); + const _b = + typeof get(b.json, field.name) === 'string' + ? (get(b.json, field.name) as string).toLowerCase() + : get(b.json, field.name); + lessThan = lt(_a, _b); + } else { + const _a = + typeof a.json[field.name] === 'string' + ? (a.json[field.name] as string).toLowerCase() + : a.json[field.name]; + const _b = + typeof b.json[field.name] === 'string' + ? (b.json[field.name] as string).toLowerCase() + : b.json[field.name]; + lessThan = lt(_a, _b); + } + if (lessThan) { + result = -1 * field.dir; + } else { + result = 1 * field.dir; + } + break; + } + } + return result; + }); + } else { + returnData = sortByCode.call(this, returnData); + } + return [returnData]; + } +} diff --git a/packages/nodes-base/nodes/Transform/Sort/sort.operation.ts b/packages/nodes-base/nodes/Transform/Sort/sort.operation.ts deleted file mode 100644 index 6825779e28a99..0000000000000 --- a/packages/nodes-base/nodes/Transform/Sort/sort.operation.ts +++ /dev/null @@ -1,275 +0,0 @@ -import type { - IExecuteFunctions, - IDataObject, - INodeExecutionData, - INodeProperties, -} from 'n8n-workflow'; -import { NodeOperationError } from 'n8n-workflow'; - -import get from 'lodash/get'; - -import isEqual from 'lodash/isEqual'; -import lt from 'lodash/lt'; - -import { shuffleArray, sortByCode } from '../../helpers/utils'; -import { disableDotNotationBoolean } from '../common.descriptions'; -import { updateDisplayOptions } from '@utils/utilities'; - -const properties: INodeProperties[] = [ - { - displayName: 'Type', - name: 'type', - type: 'options', - options: [ - { - name: 'Simple', - value: 'simple', - }, - { - name: 'Random', - value: 'random', - }, - { - name: 'Code', - value: 'code', - }, - ], - default: 'simple', - description: 'The fields of the input items to compare to see if they are the same', - }, - { - displayName: 'Fields To Sort By', - name: 'sortFieldsUi', - type: 'fixedCollection', - typeOptions: { - multipleValues: true, - }, - placeholder: 'Add Field To Sort By', - options: [ - { - displayName: '', - name: 'sortField', - values: [ - { - displayName: 'Field Name', - name: 'fieldName', - type: 'string', - required: true, - default: '', - description: 'The field to sort by', - // eslint-disable-next-line n8n-nodes-base/node-param-placeholder-miscased-id - placeholder: 'e.g. id', - hint: ' Enter the field name as text', - requiresDataPath: 'single', - }, - { - displayName: 'Order', - name: 'order', - type: 'options', - options: [ - { - name: 'Ascending', - value: 'ascending', - }, - { - name: 'Descending', - value: 'descending', - }, - ], - default: 'ascending', - description: 'The order to sort by', - }, - ], - }, - ], - default: {}, - description: 'The fields of the input items to compare to see if they are the same', - displayOptions: { - show: { - type: ['simple'], - }, - }, - }, - { - displayName: 'Code', - name: 'code', - type: 'string', - typeOptions: { - alwaysOpenEditWindow: true, - editor: 'code', - rows: 10, - }, - default: `// The two items to compare are in the variables a and b -// Access the fields in a.json and b.json -// Return -1 if a should go before b -// Return 1 if b should go before a -// Return 0 if there's no difference - -fieldName = 'myField'; - -if (a.json[fieldName] < b.json[fieldName]) { -return -1; -} -if (a.json[fieldName] > b.json[fieldName]) { -return 1; -} -return 0;`, - description: 'Javascript code to determine the order of any two items', - displayOptions: { - show: { - type: ['code'], - }, - }, - }, - { - displayName: 'Options', - name: 'options', - type: 'collection', - placeholder: 'Add Field', - default: {}, - displayOptions: { - show: { - type: ['simple'], - }, - }, - options: [disableDotNotationBoolean], - }, -]; - -const displayOptions = { - show: { - resource: ['itemList'], - operation: ['sort'], - }, -}; - -export const description = updateDisplayOptions(displayOptions, properties); - -export async function execute( - this: IExecuteFunctions, - items: INodeExecutionData[], -): Promise { - let returnData = [...items]; - const type = this.getNodeParameter('type', 0) as string; - const disableDotNotation = this.getNodeParameter( - 'options.disableDotNotation', - 0, - false, - ) as boolean; - - if (type === 'random') { - shuffleArray(returnData); - return returnData; - } - - if (type === 'simple') { - const sortFieldsUi = this.getNodeParameter('sortFieldsUi', 0) as IDataObject; - const sortFields = sortFieldsUi.sortField as Array<{ - fieldName: string; - order: 'ascending' | 'descending'; - }>; - - if (!sortFields?.length) { - throw new NodeOperationError( - this.getNode(), - 'No sorting specified. Please add a field to sort by', - ); - } - - for (const { fieldName } of sortFields) { - let found = false; - for (const item of items) { - if (!disableDotNotation) { - if (get(item.json, fieldName) !== undefined) { - found = true; - } - } else if (item.json.hasOwnProperty(fieldName)) { - found = true; - } - } - if (!found && disableDotNotation && fieldName.includes('.')) { - throw new NodeOperationError( - this.getNode(), - `Couldn't find the field '${fieldName}' in the input data`, - { - description: - "If you're trying to use a nested field, make sure you turn off 'disable dot notation' in the node options", - }, - ); - } else if (!found) { - throw new NodeOperationError( - this.getNode(), - `Couldn't find the field '${fieldName}' in the input data`, - ); - } - } - - const sortFieldsWithDirection = sortFields.map((field) => ({ - name: field.fieldName, - dir: field.order === 'ascending' ? 1 : -1, - })); - - returnData.sort((a, b) => { - let result = 0; - for (const field of sortFieldsWithDirection) { - let equal; - if (!disableDotNotation) { - const _a = - typeof get(a.json, field.name) === 'string' - ? (get(a.json, field.name) as string).toLowerCase() - : get(a.json, field.name); - const _b = - typeof get(b.json, field.name) === 'string' - ? (get(b.json, field.name) as string).toLowerCase() - : get(b.json, field.name); - equal = isEqual(_a, _b); - } else { - const _a = - typeof a.json[field.name] === 'string' - ? (a.json[field.name] as string).toLowerCase() - : a.json[field.name]; - const _b = - typeof b.json[field.name] === 'string' - ? (b.json[field.name] as string).toLowerCase() - : b.json[field.name]; - equal = isEqual(_a, _b); - } - - if (!equal) { - let lessThan; - if (!disableDotNotation) { - const _a = - typeof get(a.json, field.name) === 'string' - ? (get(a.json, field.name) as string).toLowerCase() - : get(a.json, field.name); - const _b = - typeof get(b.json, field.name) === 'string' - ? (get(b.json, field.name) as string).toLowerCase() - : get(b.json, field.name); - lessThan = lt(_a, _b); - } else { - const _a = - typeof a.json[field.name] === 'string' - ? (a.json[field.name] as string).toLowerCase() - : a.json[field.name]; - const _b = - typeof b.json[field.name] === 'string' - ? (b.json[field.name] as string).toLowerCase() - : b.json[field.name]; - lessThan = lt(_a, _b); - } - if (lessThan) { - result = -1 * field.dir; - } else { - result = 1 * field.dir; - } - break; - } - } - return result; - }); - } else { - returnData = sortByCode.call(this, returnData); - } - return returnData; -} diff --git a/packages/nodes-base/nodes/Transform/Sort/sort.svg b/packages/nodes-base/nodes/Transform/Sort/sort.svg new file mode 100644 index 0000000000000..56100defcee2d --- /dev/null +++ b/packages/nodes-base/nodes/Transform/Sort/sort.svg @@ -0,0 +1,13 @@ + + + + + + \ No newline at end of file diff --git a/packages/nodes-base/nodes/Transform/Sort/utils.ts b/packages/nodes-base/nodes/Transform/Sort/utils.ts new file mode 100644 index 0000000000000..3b156e206bb9d --- /dev/null +++ b/packages/nodes-base/nodes/Transform/Sort/utils.ts @@ -0,0 +1,31 @@ +import { NodeVM } from '@n8n/vm2'; +import { type IExecuteFunctions, type INodeExecutionData, NodeOperationError } from 'n8n-workflow'; + +export const shuffleArray = (array: any[]) => { + for (let i = array.length - 1; i > 0; i--) { + const j = Math.floor(Math.random() * (i + 1)); + [array[i], array[j]] = [array[j], array[i]]; + } +}; + +const returnRegExp = /\breturn\b/g; +export function sortByCode( + this: IExecuteFunctions, + items: INodeExecutionData[], +): INodeExecutionData[] { + const code = this.getNodeParameter('code', 0) as string; + if (!returnRegExp.test(code)) { + throw new NodeOperationError( + this.getNode(), + "Sort code doesn't return. Please add a 'return' statement to your code", + ); + } + + const mode = this.getMode(); + const vm = new NodeVM({ + console: mode === 'manual' ? 'redirect' : 'inherit', + sandbox: { items }, + }); + + return vm.run(`module.exports = items.sort((a, b) => { ${code} })`); +} diff --git a/packages/nodes-base/nodes/Transform/SplitOut/SplitOut.node.ts b/packages/nodes-base/nodes/Transform/SplitOut/SplitOut.node.ts index e69de29bb2d1d..c58af6e5369c7 100644 --- a/packages/nodes-base/nodes/Transform/SplitOut/SplitOut.node.ts +++ b/packages/nodes-base/nodes/Transform/SplitOut/SplitOut.node.ts @@ -0,0 +1,259 @@ +import get from 'lodash/get'; +import unset from 'lodash/unset'; +import { + type IBinaryData, + NodeOperationError, + deepCopy, + type IDataObject, + type IExecuteFunctions, + type INodeExecutionData, + type INodeType, + type INodeTypeDescription, +} from 'n8n-workflow'; +import { prepareFieldsArray } from '../utils/utils'; + +export class SplitOut implements INodeType { + description: INodeTypeDescription = { + displayName: 'SplitOut', + name: 'splitOut', + icon: 'file:splitOut.svg', + group: ['transform'], + subtitle: '', + version: 1, + description: 'Turn a list inside item(s) into separate items', + defaults: { + name: 'Split Out', + }, + inputs: ['main'], + outputs: ['main'], + properties: [ + { + displayName: 'Fields To Split Out', + name: 'fieldToSplitOut', + type: 'string', + default: '', + required: true, + placeholder: 'Drag fields from the left or type their names', + description: + 'The name of the input fields to break out into separate items. Separate multiple field names by commas. For binary data, use $binary.', + requiresDataPath: 'multiple', + }, + { + displayName: 'Include', + name: 'include', + type: 'options', + options: [ + { + name: 'No Other Fields', + value: 'noOtherFields', + }, + { + name: 'All Other Fields', + value: 'allOtherFields', + }, + { + name: 'Selected Other Fields', + value: 'selectedOtherFields', + }, + ], + default: 'noOtherFields', + description: 'Whether to copy any other fields into the new items', + }, + { + displayName: 'Fields To Include', + name: 'fieldsToInclude', + type: 'string', + placeholder: 'e.g. email, name', + requiresDataPath: 'multiple', + description: 'Fields in the input items to aggregate together', + default: '', + displayOptions: { + show: { + include: ['selectedOtherFields'], + }, + }, + }, + { + displayName: 'Options', + name: 'options', + type: 'collection', + placeholder: 'Add Field', + default: {}, + options: [ + { + displayName: 'Disable Dot Notation', + name: 'disableDotNotation', + type: 'boolean', + default: false, + description: + 'Whether to disallow referencing child fields using `parent.child` in the field name', + }, + { + displayName: 'Destination Field Name', + name: 'destinationFieldName', + type: 'string', + requiresDataPath: 'multiple', + default: '', + description: 'The field in the output under which to put the split field contents', + }, + { + displayName: 'Include Binary', + name: 'includeBinary', + type: 'boolean', + default: false, + description: 'Whether to include the binary data in the new items', + }, + ], + }, + ], + }; + + async execute(this: IExecuteFunctions): Promise { + const returnData: INodeExecutionData[] = []; + const items = this.getInputData(); + + for (let i = 0; i < items.length; i++) { + const fieldsToSplitOut = (this.getNodeParameter('fieldToSplitOut', i) as string) + .split(',') + .map((field) => field.trim().replace(/^\$json\./, '')); + + const options = this.getNodeParameter('options', i, {}); + + const disableDotNotation = options.disableDotNotation as boolean; + + const destinationFields = ((options.destinationFieldName as string) || '') + .split(',') + .filter((field) => field.trim() !== '') + .map((field) => field.trim()); + + if (destinationFields.length && destinationFields.length !== fieldsToSplitOut.length) { + throw new NodeOperationError( + this.getNode(), + 'If multiple fields to split out are given, the same number of destination fields must be given', + ); + } + + const include = this.getNodeParameter('include', i) as + | 'selectedOtherFields' + | 'allOtherFields' + | 'noOtherFields'; + + const multiSplit = fieldsToSplitOut.length > 1; + + const item = { ...items[i].json }; + const splited: INodeExecutionData[] = []; + for (const [entryIndex, fieldToSplitOut] of fieldsToSplitOut.entries()) { + const destinationFieldName = destinationFields[entryIndex] || ''; + + let entityToSplit: IDataObject[] = []; + + if (fieldToSplitOut === '$binary') { + entityToSplit = Object.entries(items[i].binary || {}).map(([key, value]) => ({ + [key]: value, + })); + } else { + if (!disableDotNotation) { + entityToSplit = get(item, fieldToSplitOut) as IDataObject[]; + } else { + entityToSplit = item[fieldToSplitOut] as IDataObject[]; + } + + if (entityToSplit === undefined) { + entityToSplit = []; + } + + if (typeof entityToSplit !== 'object' || entityToSplit === null) { + entityToSplit = [entityToSplit]; + } + + if (!Array.isArray(entityToSplit)) { + entityToSplit = Object.values(entityToSplit); + } + } + + for (const [elementIndex, element] of entityToSplit.entries()) { + if (splited[elementIndex] === undefined) { + splited[elementIndex] = { json: {}, pairedItem: { item: i } }; + } + + const fieldName = destinationFieldName || fieldToSplitOut; + + if (fieldToSplitOut === '$binary') { + if (splited[elementIndex].binary === undefined) { + splited[elementIndex].binary = {}; + } + splited[elementIndex].binary![Object.keys(element)[0]] = Object.values( + element, + )[0] as IBinaryData; + + continue; + } + + if (typeof element === 'object' && element !== null && include === 'noOtherFields') { + if (destinationFieldName === '' && !multiSplit) { + splited[elementIndex] = { + json: { ...splited[elementIndex].json, ...element }, + pairedItem: { item: i }, + }; + } else { + splited[elementIndex].json[fieldName] = element; + } + } else { + splited[elementIndex].json[fieldName] = element; + } + } + } + + for (const splitEntry of splited) { + let newItem: INodeExecutionData = splitEntry; + + if (include === 'allOtherFields') { + const itemCopy = deepCopy(item); + for (const fieldToSplitOut of fieldsToSplitOut) { + if (!disableDotNotation) { + unset(itemCopy, fieldToSplitOut); + } else { + delete itemCopy[fieldToSplitOut]; + } + } + newItem.json = { ...itemCopy, ...splitEntry.json }; + } + + if (include === 'selectedOtherFields') { + const fieldsToInclude = prepareFieldsArray( + this.getNodeParameter('fieldsToInclude', i, '') as string, + 'Fields To Include', + ); + + if (!fieldsToInclude.length) { + throw new NodeOperationError(this.getNode(), 'No fields specified', { + description: 'Please add a field to include', + }); + } + + for (const field of fieldsToInclude) { + if (!disableDotNotation) { + splitEntry.json[field] = get(item, field); + } else { + splitEntry.json[field] = item[field]; + } + } + + newItem = splitEntry; + } + + const includeBinary = options.includeBinary as boolean; + + if (includeBinary) { + if (items[i].binary && !newItem.binary) { + newItem.binary = items[i].binary; + } + } + + returnData.push(newItem); + } + } + + return [returnData]; + } +} diff --git a/packages/nodes-base/nodes/Transform/SplitOut/splitOut.svg b/packages/nodes-base/nodes/Transform/SplitOut/splitOut.svg new file mode 100644 index 0000000000000..56100defcee2d --- /dev/null +++ b/packages/nodes-base/nodes/Transform/SplitOut/splitOut.svg @@ -0,0 +1,13 @@ + + + + + + \ No newline at end of file diff --git a/packages/nodes-base/nodes/Transform/utils/utils.ts b/packages/nodes-base/nodes/Transform/utils/utils.ts new file mode 100644 index 0000000000000..41d64c408a48c --- /dev/null +++ b/packages/nodes-base/nodes/Transform/utils/utils.ts @@ -0,0 +1,14 @@ +export const prepareFieldsArray = (fields: string | string[], fieldName = 'Fields') => { + if (typeof fields === 'string') { + return fields + .split(',') + .map((entry) => entry.trim()) + .filter((entry) => entry !== ''); + } + if (Array.isArray(fields)) { + return fields; + } + throw new Error( + `The \'${fieldName}\' parameter must be a string of fields separated by commas or an array of strings.`, + ); +}; From 9de95d30000aaac19794609dc75e9f449d602d73 Mon Sep 17 00:00:00 2001 From: Elias Meire Date: Mon, 20 Nov 2023 14:44:55 +0100 Subject: [PATCH 04/17] Add codex for new nodes --- .../Transform/Aggregate/Aggregate.node.json | 19 + .../nodes/Transform/Limit/Limit.node.json | 19 + .../RemoveDuplicates.node.json | 29 + .../nodes/Transform/Sort/Sort.node.json | 19 + .../Transform/SplitOut/SplitOut.node.json | 19 + .../nodes/Transform/SplitOut/SplitOut.node.ts | 2 +- .../SplitOut/splitOutItems.operation.ts | 248 ------- .../Transform/Summarize/Summarize.node.json | 32 + .../Transform/Summarize/Summarize.node.ts | 350 ++++++++++ .../Summarize/summarize.operation.ts | 615 ------------------ .../nodes/Transform/Summarize/summarize.svg | 13 + .../nodes/Transform/Summarize/utils.ts | 288 ++++++++ 12 files changed, 789 insertions(+), 864 deletions(-) create mode 100644 packages/nodes-base/nodes/Transform/Aggregate/Aggregate.node.json create mode 100644 packages/nodes-base/nodes/Transform/Limit/Limit.node.json create mode 100644 packages/nodes-base/nodes/Transform/RemoveDuplicates/RemoveDuplicates.node.json create mode 100644 packages/nodes-base/nodes/Transform/Sort/Sort.node.json create mode 100644 packages/nodes-base/nodes/Transform/SplitOut/SplitOut.node.json delete mode 100644 packages/nodes-base/nodes/Transform/SplitOut/splitOutItems.operation.ts create mode 100644 packages/nodes-base/nodes/Transform/Summarize/Summarize.node.json delete mode 100644 packages/nodes-base/nodes/Transform/Summarize/summarize.operation.ts create mode 100644 packages/nodes-base/nodes/Transform/Summarize/summarize.svg create mode 100644 packages/nodes-base/nodes/Transform/Summarize/utils.ts diff --git a/packages/nodes-base/nodes/Transform/Aggregate/Aggregate.node.json b/packages/nodes-base/nodes/Transform/Aggregate/Aggregate.node.json new file mode 100644 index 0000000000000..7cf41cfed2daa --- /dev/null +++ b/packages/nodes-base/nodes/Transform/Aggregate/Aggregate.node.json @@ -0,0 +1,19 @@ +{ + "node": "n8n-nodes-base.aggregate", + "nodeVersion": "1.0", + "codexVersion": "1.0", + "details": "", + "categories": ["Core Nodes"], + "resources": { + "primaryDocumentation": [ + { + "url": "https://docs.n8n.io/integrations/builtin/core-nodes/n8n-nodes-base.aggregate/" + } + ], + "generic": [] + }, + "alias": ["Aggregate", "Combine", "Flatten", "Transform", "Array", "List", "Item"], + "subcategories": { + "Core Nodes": ["Helpers", "Data Transformation"] + } +} diff --git a/packages/nodes-base/nodes/Transform/Limit/Limit.node.json b/packages/nodes-base/nodes/Transform/Limit/Limit.node.json new file mode 100644 index 0000000000000..68faf8da85f01 --- /dev/null +++ b/packages/nodes-base/nodes/Transform/Limit/Limit.node.json @@ -0,0 +1,19 @@ +{ + "node": "n8n-nodes-base.limit", + "nodeVersion": "1.0", + "codexVersion": "1.0", + "details": "", + "categories": ["Core Nodes"], + "resources": { + "primaryDocumentation": [ + { + "url": "https://docs.n8n.io/integrations/builtin/core-nodes/n8n-nodes-base.limit/" + } + ], + "generic": [] + }, + "alias": ["Limit", "Remove", "Slice", "Transform", "Array", "List", "Item"], + "subcategories": { + "Core Nodes": ["Helpers", "Data Transformation"] + } +} diff --git a/packages/nodes-base/nodes/Transform/RemoveDuplicates/RemoveDuplicates.node.json b/packages/nodes-base/nodes/Transform/RemoveDuplicates/RemoveDuplicates.node.json new file mode 100644 index 0000000000000..5db54477b4c80 --- /dev/null +++ b/packages/nodes-base/nodes/Transform/RemoveDuplicates/RemoveDuplicates.node.json @@ -0,0 +1,29 @@ +{ + "node": "n8n-nodes-base.removeDuplicates", + "nodeVersion": "1.0", + "codexVersion": "1.0", + "details": "", + "categories": ["Core Nodes"], + "resources": { + "primaryDocumentation": [ + { + "url": "https://docs.n8n.io/integrations/builtin/core-nodes/n8n-nodes-base.removeDuplicates/" + } + ], + "generic": [] + }, + "alias": [ + "Dedupe", + "Deduplicate", + "Duplicates", + "Remove", + "Unique", + "Transform", + "Array", + "List", + "Item" + ], + "subcategories": { + "Core Nodes": ["Helpers", "Data Transformation"] + } +} diff --git a/packages/nodes-base/nodes/Transform/Sort/Sort.node.json b/packages/nodes-base/nodes/Transform/Sort/Sort.node.json new file mode 100644 index 0000000000000..88a714a7636a8 --- /dev/null +++ b/packages/nodes-base/nodes/Transform/Sort/Sort.node.json @@ -0,0 +1,19 @@ +{ + "node": "n8n-nodes-base.sort", + "nodeVersion": "1.0", + "codexVersion": "1.0", + "details": "", + "categories": ["Core Nodes"], + "resources": { + "primaryDocumentation": [ + { + "url": "https://docs.n8n.io/integrations/builtin/core-nodes/n8n-nodes-base.sort/" + } + ], + "generic": [] + }, + "alias": ["Sort", "Order", "Transform", "Array", "List", "Item"], + "subcategories": { + "Core Nodes": ["Helpers", "Data Transformation"] + } +} diff --git a/packages/nodes-base/nodes/Transform/SplitOut/SplitOut.node.json b/packages/nodes-base/nodes/Transform/SplitOut/SplitOut.node.json new file mode 100644 index 0000000000000..05677d6c8390e --- /dev/null +++ b/packages/nodes-base/nodes/Transform/SplitOut/SplitOut.node.json @@ -0,0 +1,19 @@ +{ + "node": "n8n-nodes-base.splitOut", + "nodeVersion": "1.0", + "codexVersion": "1.0", + "details": "", + "categories": ["Core Nodes"], + "resources": { + "primaryDocumentation": [ + { + "url": "https://docs.n8n.io/integrations/builtin/core-nodes/n8n-nodes-base.splitOut/" + } + ], + "generic": [] + }, + "alias": ["Split", "Nested", "Transform", "Array", "List", "Item"], + "subcategories": { + "Core Nodes": ["Helpers", "Data Transformation"] + } +} diff --git a/packages/nodes-base/nodes/Transform/SplitOut/SplitOut.node.ts b/packages/nodes-base/nodes/Transform/SplitOut/SplitOut.node.ts index c58af6e5369c7..5a48662cb903c 100644 --- a/packages/nodes-base/nodes/Transform/SplitOut/SplitOut.node.ts +++ b/packages/nodes-base/nodes/Transform/SplitOut/SplitOut.node.ts @@ -14,7 +14,7 @@ import { prepareFieldsArray } from '../utils/utils'; export class SplitOut implements INodeType { description: INodeTypeDescription = { - displayName: 'SplitOut', + displayName: 'Split Out', name: 'splitOut', icon: 'file:splitOut.svg', group: ['transform'], diff --git a/packages/nodes-base/nodes/Transform/SplitOut/splitOutItems.operation.ts b/packages/nodes-base/nodes/Transform/SplitOut/splitOutItems.operation.ts deleted file mode 100644 index 13d5bd89189b2..0000000000000 --- a/packages/nodes-base/nodes/Transform/SplitOut/splitOutItems.operation.ts +++ /dev/null @@ -1,248 +0,0 @@ -import type { - IBinaryData, - IDataObject, - IExecuteFunctions, - INodeExecutionData, - INodeProperties, -} from 'n8n-workflow'; -import { deepCopy, NodeOperationError } from 'n8n-workflow'; - -import get from 'lodash/get'; -import unset from 'lodash/unset'; -import { disableDotNotationBoolean } from '../common.descriptions'; -import { prepareFieldsArray } from '../../helpers/utils'; -import { updateDisplayOptions } from '@utils/utilities'; - -const properties: INodeProperties[] = [ - { - displayName: 'Fields To Split Out', - name: 'fieldToSplitOut', - type: 'string', - default: '', - required: true, - placeholder: 'Drag fields from the left or type their names', - description: - 'The name of the input fields to break out into separate items. Separate multiple field names by commas. For binary data, use $binary.', - requiresDataPath: 'multiple', - }, - { - displayName: 'Include', - name: 'include', - type: 'options', - options: [ - { - name: 'No Other Fields', - value: 'noOtherFields', - }, - { - name: 'All Other Fields', - value: 'allOtherFields', - }, - { - name: 'Selected Other Fields', - value: 'selectedOtherFields', - }, - ], - default: 'noOtherFields', - description: 'Whether to copy any other fields into the new items', - }, - { - displayName: 'Fields To Include', - name: 'fieldsToInclude', - type: 'string', - placeholder: 'e.g. email, name', - requiresDataPath: 'multiple', - description: 'Fields in the input items to aggregate together', - default: '', - displayOptions: { - show: { - include: ['selectedOtherFields'], - }, - }, - }, - { - displayName: 'Options', - name: 'options', - type: 'collection', - placeholder: 'Add Field', - default: {}, - options: [ - disableDotNotationBoolean, - { - displayName: 'Destination Field Name', - name: 'destinationFieldName', - type: 'string', - requiresDataPath: 'multiple', - default: '', - description: 'The field in the output under which to put the split field contents', - }, - { - displayName: 'Include Binary', - name: 'includeBinary', - type: 'boolean', - default: false, - description: 'Whether to include the binary data in the new items', - }, - ], - }, -]; - -const displayOptions = { - show: { - resource: ['itemList'], - operation: ['splitOutItems'], - }, -}; - -export const description = updateDisplayOptions(displayOptions, properties); - -export async function execute( - this: IExecuteFunctions, - items: INodeExecutionData[], -): Promise { - const returnData: INodeExecutionData[] = []; - - for (let i = 0; i < items.length; i++) { - const fieldsToSplitOut = (this.getNodeParameter('fieldToSplitOut', i) as string) - .split(',') - .map((field) => field.trim().replace(/^\$json\./, '')); - - const options = this.getNodeParameter('options', i, {}); - - const disableDotNotation = options.disableDotNotation as boolean; - - const destinationFields = ((options.destinationFieldName as string) || '') - .split(',') - .filter((field) => field.trim() !== '') - .map((field) => field.trim()); - - if (destinationFields.length && destinationFields.length !== fieldsToSplitOut.length) { - throw new NodeOperationError( - this.getNode(), - 'If multiple fields to split out are given, the same number of destination fields must be given', - ); - } - - const include = this.getNodeParameter('include', i) as - | 'selectedOtherFields' - | 'allOtherFields' - | 'noOtherFields'; - - const multiSplit = fieldsToSplitOut.length > 1; - - const item = { ...items[i].json }; - const splited: INodeExecutionData[] = []; - for (const [entryIndex, fieldToSplitOut] of fieldsToSplitOut.entries()) { - const destinationFieldName = destinationFields[entryIndex] || ''; - - let entityToSplit: IDataObject[] = []; - - if (fieldToSplitOut === '$binary') { - entityToSplit = Object.entries(items[i].binary || {}).map(([key, value]) => ({ - [key]: value, - })); - } else { - if (!disableDotNotation) { - entityToSplit = get(item, fieldToSplitOut) as IDataObject[]; - } else { - entityToSplit = item[fieldToSplitOut] as IDataObject[]; - } - - if (entityToSplit === undefined) { - entityToSplit = []; - } - - if (typeof entityToSplit !== 'object' || entityToSplit === null) { - entityToSplit = [entityToSplit]; - } - - if (!Array.isArray(entityToSplit)) { - entityToSplit = Object.values(entityToSplit); - } - } - - for (const [elementIndex, element] of entityToSplit.entries()) { - if (splited[elementIndex] === undefined) { - splited[elementIndex] = { json: {}, pairedItem: { item: i } }; - } - - const fieldName = destinationFieldName || fieldToSplitOut; - - if (fieldToSplitOut === '$binary') { - if (splited[elementIndex].binary === undefined) { - splited[elementIndex].binary = {}; - } - splited[elementIndex].binary![Object.keys(element)[0]] = Object.values( - element, - )[0] as IBinaryData; - - continue; - } - - if (typeof element === 'object' && element !== null && include === 'noOtherFields') { - if (destinationFieldName === '' && !multiSplit) { - splited[elementIndex] = { - json: { ...splited[elementIndex].json, ...element }, - pairedItem: { item: i }, - }; - } else { - splited[elementIndex].json[fieldName] = element; - } - } else { - splited[elementIndex].json[fieldName] = element; - } - } - } - - for (const splitEntry of splited) { - let newItem: INodeExecutionData = splitEntry; - - if (include === 'allOtherFields') { - const itemCopy = deepCopy(item); - for (const fieldToSplitOut of fieldsToSplitOut) { - if (!disableDotNotation) { - unset(itemCopy, fieldToSplitOut); - } else { - delete itemCopy[fieldToSplitOut]; - } - } - newItem.json = { ...itemCopy, ...splitEntry.json }; - } - - if (include === 'selectedOtherFields') { - const fieldsToInclude = prepareFieldsArray( - this.getNodeParameter('fieldsToInclude', i, '') as string, - 'Fields To Include', - ); - - if (!fieldsToInclude.length) { - throw new NodeOperationError(this.getNode(), 'No fields specified', { - description: 'Please add a field to include', - }); - } - - for (const field of fieldsToInclude) { - if (!disableDotNotation) { - splitEntry.json[field] = get(item, field); - } else { - splitEntry.json[field] = item[field]; - } - } - - newItem = splitEntry; - } - - const includeBinary = options.includeBinary as boolean; - - if (includeBinary) { - if (items[i].binary && !newItem.binary) { - newItem.binary = items[i].binary; - } - } - - returnData.push(newItem); - } - } - - return returnData; -} diff --git a/packages/nodes-base/nodes/Transform/Summarize/Summarize.node.json b/packages/nodes-base/nodes/Transform/Summarize/Summarize.node.json new file mode 100644 index 0000000000000..6417c4b18b52a --- /dev/null +++ b/packages/nodes-base/nodes/Transform/Summarize/Summarize.node.json @@ -0,0 +1,32 @@ +{ + "node": "n8n-nodes-base.summarize", + "nodeVersion": "1.0", + "codexVersion": "1.0", + "details": "", + "categories": ["Core Nodes"], + "resources": { + "primaryDocumentation": [ + { + "url": "https://docs.n8n.io/integrations/builtin/core-nodes/n8n-nodes-base.summarize/" + } + ], + "generic": [] + }, + "alias": [ + "Summarise", + "Summarize", + "Group", + "Pivot", + "Sum", + "Count", + "Min", + "Max", + "Transform", + "Array", + "List", + "Item" + ], + "subcategories": { + "Core Nodes": ["Helpers", "Data Transformation"] + } +} diff --git a/packages/nodes-base/nodes/Transform/Summarize/Summarize.node.ts b/packages/nodes-base/nodes/Transform/Summarize/Summarize.node.ts index e69de29bb2d1d..1eadc629bb757 100644 --- a/packages/nodes-base/nodes/Transform/Summarize/Summarize.node.ts +++ b/packages/nodes-base/nodes/Transform/Summarize/Summarize.node.ts @@ -0,0 +1,350 @@ +import { + NodeOperationError, + type IExecuteFunctions, + type INodeExecutionData, + type INodeType, + type INodeTypeDescription, +} from 'n8n-workflow'; +import { + type Aggregations, + NUMERICAL_AGGREGATIONS, + type SummarizeOptions, + aggregationToArray, + checkIfFieldExists, + fieldValueGetter, + splitData, +} from './utils'; + +export class Summarize implements INodeType { + description: INodeTypeDescription = { + displayName: 'Summarize', + name: 'summarize', + icon: 'file:summarize.svg', + group: ['transform'], + subtitle: '', + version: 1, + description: 'Sum, count, max, etc. across items', + defaults: { + name: 'Summarize', + }, + inputs: ['main'], + outputs: ['main'], + properties: [ + { + displayName: 'Fields to Summarize', + name: 'fieldsToSummarize', + type: 'fixedCollection', + placeholder: 'Add Field', + default: { values: [{ aggregation: 'count', field: '' }] }, + typeOptions: { + multipleValues: true, + }, + options: [ + { + displayName: '', + name: 'values', + values: [ + { + displayName: 'Aggregation', + name: 'aggregation', + type: 'options', + options: [ + { + name: 'Append', + value: 'append', + }, + { + name: 'Average', + value: 'average', + }, + { + name: 'Concatenate', + value: 'concatenate', + }, + { + name: 'Count', + value: 'count', + }, + { + name: 'Count Unique', + value: 'countUnique', + }, + { + name: 'Max', + value: 'max', + }, + { + name: 'Min', + value: 'min', + }, + { + name: 'Sum', + value: 'sum', + }, + ], + default: 'count', + description: 'How to combine the values of the field you want to summarize', + }, + //field repeated to have different descriptions for different aggregations -------------------------------- + { + displayName: 'Field', + name: 'field', + type: 'string', + default: '', + description: 'The name of an input field that you want to summarize', + placeholder: 'e.g. cost', + hint: ' Enter the field name as text', + displayOptions: { + hide: { + aggregation: [...NUMERICAL_AGGREGATIONS, 'countUnique', 'count', 'max', 'min'], + }, + }, + requiresDataPath: 'single', + }, + { + displayName: 'Field', + name: 'field', + type: 'string', + default: '', + description: + 'The name of an input field that you want to summarize. The field should contain numerical values; null, undefined, empty strings would be ignored.', + placeholder: 'e.g. cost', + hint: ' Enter the field name as text', + displayOptions: { + show: { + aggregation: NUMERICAL_AGGREGATIONS, + }, + }, + requiresDataPath: 'single', + }, + { + displayName: 'Field', + name: 'field', + type: 'string', + default: '', + description: + 'The name of an input field that you want to summarize; null, undefined, empty strings would be ignored', + placeholder: 'e.g. cost', + hint: ' Enter the field name as text', + displayOptions: { + show: { + aggregation: ['countUnique', 'count', 'max', 'min'], + }, + }, + requiresDataPath: 'single', + }, + // ---------------------------------------------------------------------------------------------------------- + { + displayName: 'Include Empty Values', + name: 'includeEmpty', + type: 'boolean', + default: false, + displayOptions: { + show: { + aggregation: ['append', 'concatenate'], + }, + }, + }, + { + displayName: 'Separator', + name: 'separateBy', + type: 'options', + default: ',', + // eslint-disable-next-line n8n-nodes-base/node-param-options-type-unsorted-items + options: [ + { + name: 'Comma', + value: ',', + }, + { + name: 'Comma and Space', + value: ', ', + }, + { + name: 'New Line', + value: '\n', + }, + { + name: 'None', + value: '', + }, + { + name: 'Space', + value: ' ', + }, + { + name: 'Other', + value: 'other', + }, + ], + hint: 'What to insert between values', + displayOptions: { + show: { + aggregation: ['concatenate'], + }, + }, + }, + { + displayName: 'Custom Separator', + name: 'customSeparator', + type: 'string', + default: '', + displayOptions: { + show: { + aggregation: ['concatenate'], + separateBy: ['other'], + }, + }, + }, + ], + }, + ], + }, + // fieldsToSplitBy repeated to have different displayName for singleItem and separateItems ----------------------------- + { + displayName: 'Fields to Split By', + name: 'fieldsToSplitBy', + type: 'string', + placeholder: 'e.g. country, city', + default: '', + description: 'The name of the input fields that you want to split the summary by', + hint: 'Enter the name of the fields as text (separated by commas)', + displayOptions: { + hide: { + '/options.outputFormat': ['singleItem'], + }, + }, + requiresDataPath: 'multiple', + }, + { + displayName: 'Fields to Group By', + name: 'fieldsToSplitBy', + type: 'string', + placeholder: 'e.g. country, city', + default: '', + description: 'The name of the input fields that you want to split the summary by', + hint: 'Enter the name of the fields as text (separated by commas)', + displayOptions: { + show: { + '/options.outputFormat': ['singleItem'], + }, + }, + requiresDataPath: 'multiple', + }, + // ---------------------------------------------------------------------------------------------------------- + { + displayName: 'Options', + name: 'options', + type: 'collection', + placeholder: 'Add Option', + default: {}, + options: [ + { + displayName: 'Disable Dot Notation', + name: 'disableDotNotation', + type: 'boolean', + default: false, + description: + 'Whether to disallow referencing child fields using `parent.child` in the field name', + }, + { + displayName: 'Output Format', + name: 'outputFormat', + type: 'options', + default: 'separateItems', + options: [ + { + name: 'Each Split in a Separate Item', + value: 'separateItems', + }, + { + name: 'All Splits in a Single Item', + value: 'singleItem', + }, + ], + }, + { + // eslint-disable-next-line n8n-nodes-base/node-param-display-name-miscased + displayName: 'Ignore items without valid fields to group by', + name: 'skipEmptySplitFields', + type: 'boolean', + default: false, + }, + ], + }, + ], + }; + + async execute(this: IExecuteFunctions): Promise { + const items = this.getInputData(); + const newItems = items.map(({ json }, i) => ({ ...json, _itemIndex: i })); + + const options = this.getNodeParameter('options', 0, {}) as SummarizeOptions; + + const fieldsToSplitBy = (this.getNodeParameter('fieldsToSplitBy', 0, '') as string) + .split(',') + .map((field) => field.trim()) + .filter((field) => field); + + const fieldsToSummarize = this.getNodeParameter( + 'fieldsToSummarize.values', + 0, + [], + ) as Aggregations; + + if (fieldsToSummarize.filter((aggregation) => aggregation.field !== '').length === 0) { + throw new NodeOperationError( + this.getNode(), + "You need to add at least one aggregation to 'Fields to Summarize' with non empty 'Field'", + ); + } + + const getValue = fieldValueGetter(options.disableDotNotation); + + const nodeVersion = this.getNode().typeVersion; + + if (nodeVersion < 2.1) { + checkIfFieldExists.call(this, newItems, fieldsToSummarize, getValue); + } + + const aggregationResult = splitData( + fieldsToSplitBy, + newItems, + fieldsToSummarize, + options, + getValue, + ); + + if (options.outputFormat === 'singleItem') { + const executionData: INodeExecutionData = { + json: aggregationResult, + pairedItem: newItems.map((_v, index) => ({ + item: index, + })), + }; + return [[executionData]]; + } else { + if (!fieldsToSplitBy.length) { + const { pairedItems, ...json } = aggregationResult; + const executionData: INodeExecutionData = { + json, + pairedItem: ((pairedItems as number[]) || []).map((index: number) => ({ + item: index, + })), + }; + return [[executionData]]; + } + const returnData = aggregationToArray(aggregationResult, fieldsToSplitBy); + const executionData = returnData.map((item) => { + const { pairedItems, ...json } = item; + return { + json, + pairedItem: ((pairedItems as number[]) || []).map((index: number) => ({ + item: index, + })), + }; + }); + return [executionData]; + } + } +} diff --git a/packages/nodes-base/nodes/Transform/Summarize/summarize.operation.ts b/packages/nodes-base/nodes/Transform/Summarize/summarize.operation.ts deleted file mode 100644 index 059446ac0f19a..0000000000000 --- a/packages/nodes-base/nodes/Transform/Summarize/summarize.operation.ts +++ /dev/null @@ -1,615 +0,0 @@ -import type { - GenericValue, - IDataObject, - IExecuteFunctions, - INodeExecutionData, - INodeProperties, -} from 'n8n-workflow'; -import { NodeOperationError } from 'n8n-workflow'; - -import get from 'lodash/get'; -import { disableDotNotationBoolean } from '../common.descriptions'; -import { updateDisplayOptions } from '@utils/utilities'; - -type AggregationType = - | 'append' - | 'average' - | 'concatenate' - | 'count' - | 'countUnique' - | 'max' - | 'min' - | 'sum'; - -type Aggregation = { - aggregation: AggregationType; - field: string; - includeEmpty?: boolean; - separateBy?: string; - customSeparator?: string; -}; - -type Aggregations = Aggregation[]; - -const AggregationDisplayNames = { - append: 'appended_', - average: 'average_', - concatenate: 'concatenated_', - count: 'count_', - countUnique: 'unique_count_', - max: 'max_', - min: 'min_', - sum: 'sum_', -}; - -const NUMERICAL_AGGREGATIONS = ['average', 'sum']; - -type SummarizeOptions = { - disableDotNotation?: boolean; - outputFormat?: 'separateItems' | 'singleItem'; - skipEmptySplitFields?: boolean; -}; - -type ValueGetterFn = ( - item: IDataObject, - field: string, -) => IDataObject | IDataObject[] | GenericValue | GenericValue[]; - -export const properties: INodeProperties[] = [ - { - displayName: 'Fields to Summarize', - name: 'fieldsToSummarize', - type: 'fixedCollection', - placeholder: 'Add Field', - default: { values: [{ aggregation: 'count', field: '' }] }, - typeOptions: { - multipleValues: true, - }, - options: [ - { - displayName: '', - name: 'values', - values: [ - { - displayName: 'Aggregation', - name: 'aggregation', - type: 'options', - options: [ - { - name: 'Append', - value: 'append', - }, - { - name: 'Average', - value: 'average', - }, - { - name: 'Concatenate', - value: 'concatenate', - }, - { - name: 'Count', - value: 'count', - }, - { - name: 'Count Unique', - value: 'countUnique', - }, - { - name: 'Max', - value: 'max', - }, - { - name: 'Min', - value: 'min', - }, - { - name: 'Sum', - value: 'sum', - }, - ], - default: 'count', - description: 'How to combine the values of the field you want to summarize', - }, - //field repeated to have different descriptions for different aggregations -------------------------------- - { - displayName: 'Field', - name: 'field', - type: 'string', - default: '', - description: 'The name of an input field that you want to summarize', - placeholder: 'e.g. cost', - hint: ' Enter the field name as text', - displayOptions: { - hide: { - aggregation: [...NUMERICAL_AGGREGATIONS, 'countUnique', 'count', 'max', 'min'], - }, - }, - requiresDataPath: 'single', - }, - { - displayName: 'Field', - name: 'field', - type: 'string', - default: '', - description: - 'The name of an input field that you want to summarize. The field should contain numerical values; null, undefined, empty strings would be ignored.', - placeholder: 'e.g. cost', - hint: ' Enter the field name as text', - displayOptions: { - show: { - aggregation: NUMERICAL_AGGREGATIONS, - }, - }, - requiresDataPath: 'single', - }, - { - displayName: 'Field', - name: 'field', - type: 'string', - default: '', - description: - 'The name of an input field that you want to summarize; null, undefined, empty strings would be ignored', - placeholder: 'e.g. cost', - hint: ' Enter the field name as text', - displayOptions: { - show: { - aggregation: ['countUnique', 'count', 'max', 'min'], - }, - }, - requiresDataPath: 'single', - }, - // ---------------------------------------------------------------------------------------------------------- - { - displayName: 'Include Empty Values', - name: 'includeEmpty', - type: 'boolean', - default: false, - displayOptions: { - show: { - aggregation: ['append', 'concatenate'], - }, - }, - }, - { - displayName: 'Separator', - name: 'separateBy', - type: 'options', - default: ',', - // eslint-disable-next-line n8n-nodes-base/node-param-options-type-unsorted-items - options: [ - { - name: 'Comma', - value: ',', - }, - { - name: 'Comma and Space', - value: ', ', - }, - { - name: 'New Line', - value: '\n', - }, - { - name: 'None', - value: '', - }, - { - name: 'Space', - value: ' ', - }, - { - name: 'Other', - value: 'other', - }, - ], - hint: 'What to insert between values', - displayOptions: { - show: { - aggregation: ['concatenate'], - }, - }, - }, - { - displayName: 'Custom Separator', - name: 'customSeparator', - type: 'string', - default: '', - displayOptions: { - show: { - aggregation: ['concatenate'], - separateBy: ['other'], - }, - }, - }, - ], - }, - ], - }, - // fieldsToSplitBy repeated to have different displayName for singleItem and separateItems ----------------------------- - { - displayName: 'Fields to Split By', - name: 'fieldsToSplitBy', - type: 'string', - placeholder: 'e.g. country, city', - default: '', - description: 'The name of the input fields that you want to split the summary by', - hint: 'Enter the name of the fields as text (separated by commas)', - displayOptions: { - hide: { - '/options.outputFormat': ['singleItem'], - }, - }, - requiresDataPath: 'multiple', - }, - { - displayName: 'Fields to Group By', - name: 'fieldsToSplitBy', - type: 'string', - placeholder: 'e.g. country, city', - default: '', - description: 'The name of the input fields that you want to split the summary by', - hint: 'Enter the name of the fields as text (separated by commas)', - displayOptions: { - show: { - '/options.outputFormat': ['singleItem'], - }, - }, - requiresDataPath: 'multiple', - }, - // ---------------------------------------------------------------------------------------------------------- - { - displayName: 'Options', - name: 'options', - type: 'collection', - placeholder: 'Add Option', - default: {}, - options: [ - disableDotNotationBoolean, - { - displayName: 'Output Format', - name: 'outputFormat', - type: 'options', - default: 'separateItems', - options: [ - { - name: 'Each Split in a Separate Item', - value: 'separateItems', - }, - { - name: 'All Splits in a Single Item', - value: 'singleItem', - }, - ], - }, - { - // eslint-disable-next-line n8n-nodes-base/node-param-display-name-miscased - displayName: 'Ignore items without valid fields to group by', - name: 'skipEmptySplitFields', - type: 'boolean', - default: false, - }, - ], - }, -]; - -const displayOptions = { - show: { - resource: ['itemList'], - operation: ['summarize'], - }, -}; - -export const description = updateDisplayOptions(displayOptions, properties); - -function isEmpty(value: T) { - return value === undefined || value === null || value === ''; -} - -function parseReturnData(returnData: IDataObject) { - const regexBrackets = /[\]\["]/g; - const regexSpaces = /[ .]/g; - for (const key of Object.keys(returnData)) { - if (key.match(regexBrackets)) { - const newKey = key.replace(regexBrackets, ''); - returnData[newKey] = returnData[key]; - delete returnData[key]; - } - } - for (const key of Object.keys(returnData)) { - if (key.match(regexSpaces)) { - const newKey = key.replace(regexSpaces, '_'); - returnData[newKey] = returnData[key]; - delete returnData[key]; - } - } -} - -function parseFieldName(fieldName: string[]) { - const regexBrackets = /[\]\["]/g; - const regexSpaces = /[ .]/g; - fieldName = fieldName.map((field) => { - field = field.replace(regexBrackets, ''); - field = field.replace(regexSpaces, '_'); - return field; - }); - return fieldName; -} - -const fieldValueGetter = (disableDotNotation?: boolean) => { - if (disableDotNotation) { - return (item: IDataObject, field: string) => item[field]; - } else { - return (item: IDataObject, field: string) => get(item, field); - } -}; - -function checkIfFieldExists( - this: IExecuteFunctions, - items: IDataObject[], - aggregations: Aggregations, - getValue: ValueGetterFn, -) { - for (const aggregation of aggregations) { - if (aggregation.field === '') { - continue; - } - const exist = items.some((item) => getValue(item, aggregation.field) !== undefined); - if (!exist) { - throw new NodeOperationError( - this.getNode(), - `The field '${aggregation.field}' does not exist in any items`, - ); - } - } -} - -function aggregate(items: IDataObject[], entry: Aggregation, getValue: ValueGetterFn) { - const { aggregation, field } = entry; - let data = [...items]; - - if (NUMERICAL_AGGREGATIONS.includes(aggregation)) { - data = data.filter( - (item) => typeof getValue(item, field) === 'number' && !isEmpty(getValue(item, field)), - ); - } - - switch (aggregation) { - //combine operations - case 'append': - if (!entry.includeEmpty) { - data = data.filter((item) => !isEmpty(getValue(item, field))); - } - return data.map((item) => getValue(item, field)); - case 'concatenate': - const separateBy = entry.separateBy === 'other' ? entry.customSeparator : entry.separateBy; - if (!entry.includeEmpty) { - data = data.filter((item) => !isEmpty(getValue(item, field))); - } - return data - .map((item) => { - let value = getValue(item, field); - if (typeof value === 'object') { - value = JSON.stringify(value); - } - if (typeof value === 'undefined') { - value = 'undefined'; - } - - return value; - }) - .join(separateBy); - - //numerical operations - case 'average': - return ( - data.reduce((acc, item) => { - return acc + (getValue(item, field) as number); - }, 0) / data.length - ); - case 'sum': - return data.reduce((acc, item) => { - return acc + (getValue(item, field) as number); - }, 0); - //comparison operations - case 'min': - let min; - for (const item of data) { - const value = getValue(item, field); - if (value !== undefined && value !== null && value !== '') { - if (min === undefined || value < min) { - min = value; - } - } - } - return min !== undefined ? min : null; - case 'max': - let max; - for (const item of data) { - const value = getValue(item, field); - if (value !== undefined && value !== null && value !== '') { - if (max === undefined || value > max) { - max = value; - } - } - } - return max !== undefined ? max : null; - - //count operations - case 'countUnique': - return new Set(data.map((item) => getValue(item, field)).filter((item) => !isEmpty(item))) - .size; - default: - //count by default - return data.filter((item) => !isEmpty(getValue(item, field))).length; - } -} - -function aggregateData( - data: IDataObject[], - fieldsToSummarize: Aggregations, - options: SummarizeOptions, - getValue: ValueGetterFn, -) { - const returnData = fieldsToSummarize.reduce((acc, aggregation) => { - acc[`${AggregationDisplayNames[aggregation.aggregation]}${aggregation.field}`] = aggregate( - data, - aggregation, - getValue, - ); - return acc; - }, {} as IDataObject); - parseReturnData(returnData); - if (options.outputFormat === 'singleItem') { - parseReturnData(returnData); - return returnData; - } else { - return { ...returnData, pairedItems: data.map((item) => item._itemIndex as number) }; - } -} - -function splitData( - splitKeys: string[], - data: IDataObject[], - fieldsToSummarize: Aggregations, - options: SummarizeOptions, - getValue: ValueGetterFn, -) { - if (!splitKeys || splitKeys.length === 0) { - return aggregateData(data, fieldsToSummarize, options, getValue); - } - - const [firstSplitKey, ...restSplitKeys] = splitKeys; - - const groupedData = data.reduce((acc, item) => { - let keyValuee = getValue(item, firstSplitKey) as string; - - if (typeof keyValuee === 'object') { - keyValuee = JSON.stringify(keyValuee); - } - - if (options.skipEmptySplitFields && typeof keyValuee !== 'number' && !keyValuee) { - return acc; - } - - if (acc[keyValuee] === undefined) { - acc[keyValuee] = [item]; - } else { - (acc[keyValuee] as IDataObject[]).push(item); - } - return acc; - }, {} as IDataObject); - - return Object.keys(groupedData).reduce((acc, key) => { - const value = groupedData[key] as IDataObject[]; - acc[key] = splitData(restSplitKeys, value, fieldsToSummarize, options, getValue); - return acc; - }, {} as IDataObject); -} - -function aggregationToArray( - aggregationResult: IDataObject, - fieldsToSplitBy: string[], - previousStage: IDataObject = {}, -) { - const returnData: IDataObject[] = []; - fieldsToSplitBy = parseFieldName(fieldsToSplitBy); - const splitFieldName = fieldsToSplitBy[0]; - const isNext = fieldsToSplitBy[1]; - - if (isNext === undefined) { - for (const fieldName of Object.keys(aggregationResult)) { - returnData.push({ - ...previousStage, - [splitFieldName]: fieldName, - ...(aggregationResult[fieldName] as IDataObject), - }); - } - return returnData; - } else { - for (const key of Object.keys(aggregationResult)) { - returnData.push( - ...aggregationToArray(aggregationResult[key] as IDataObject, fieldsToSplitBy.slice(1), { - ...previousStage, - [splitFieldName]: key, - }), - ); - } - return returnData; - } -} - -export async function execute( - this: IExecuteFunctions, - items: INodeExecutionData[], -): Promise { - const newItems = items.map(({ json }, i) => ({ ...json, _itemIndex: i })); - - const options = this.getNodeParameter('options', 0, {}) as SummarizeOptions; - - const fieldsToSplitBy = (this.getNodeParameter('fieldsToSplitBy', 0, '') as string) - .split(',') - .map((field) => field.trim()) - .filter((field) => field); - - const fieldsToSummarize = this.getNodeParameter( - 'fieldsToSummarize.values', - 0, - [], - ) as Aggregations; - - if (fieldsToSummarize.filter((aggregation) => aggregation.field !== '').length === 0) { - throw new NodeOperationError( - this.getNode(), - "You need to add at least one aggregation to 'Fields to Summarize' with non empty 'Field'", - ); - } - - const getValue = fieldValueGetter(options.disableDotNotation); - - const nodeVersion = this.getNode().typeVersion; - - if (nodeVersion < 2.1) { - checkIfFieldExists.call(this, newItems, fieldsToSummarize, getValue); - } - - const aggregationResult = splitData( - fieldsToSplitBy, - newItems, - fieldsToSummarize, - options, - getValue, - ); - - if (options.outputFormat === 'singleItem') { - const executionData: INodeExecutionData = { - json: aggregationResult, - pairedItem: newItems.map((_v, index) => ({ - item: index, - })), - }; - return [executionData]; - } else { - if (!fieldsToSplitBy.length) { - const { pairedItems, ...json } = aggregationResult; - const executionData: INodeExecutionData = { - json, - pairedItem: ((pairedItems as number[]) || []).map((index: number) => ({ - item: index, - })), - }; - return [executionData]; - } - const returnData = aggregationToArray(aggregationResult, fieldsToSplitBy); - const executionData = returnData.map((item) => { - const { pairedItems, ...json } = item; - return { - json, - pairedItem: ((pairedItems as number[]) || []).map((index: number) => ({ - item: index, - })), - }; - }); - return executionData; - } -} diff --git a/packages/nodes-base/nodes/Transform/Summarize/summarize.svg b/packages/nodes-base/nodes/Transform/Summarize/summarize.svg new file mode 100644 index 0000000000000..56100defcee2d --- /dev/null +++ b/packages/nodes-base/nodes/Transform/Summarize/summarize.svg @@ -0,0 +1,13 @@ + + + + + + \ No newline at end of file diff --git a/packages/nodes-base/nodes/Transform/Summarize/utils.ts b/packages/nodes-base/nodes/Transform/Summarize/utils.ts new file mode 100644 index 0000000000000..89fdb166008c0 --- /dev/null +++ b/packages/nodes-base/nodes/Transform/Summarize/utils.ts @@ -0,0 +1,288 @@ +import get from 'lodash/get'; +import { + type IDataObject, + type GenericValue, + type IExecuteFunctions, + NodeOperationError, +} from 'n8n-workflow'; + +type AggregationType = + | 'append' + | 'average' + | 'concatenate' + | 'count' + | 'countUnique' + | 'max' + | 'min' + | 'sum'; + +export type Aggregation = { + aggregation: AggregationType; + field: string; + includeEmpty?: boolean; + separateBy?: string; + customSeparator?: string; +}; + +export type Aggregations = Aggregation[]; + +const AggregationDisplayNames = { + append: 'appended_', + average: 'average_', + concatenate: 'concatenated_', + count: 'count_', + countUnique: 'unique_count_', + max: 'max_', + min: 'min_', + sum: 'sum_', +}; + +export const NUMERICAL_AGGREGATIONS = ['average', 'sum']; + +export type SummarizeOptions = { + disableDotNotation?: boolean; + outputFormat?: 'separateItems' | 'singleItem'; + skipEmptySplitFields?: boolean; +}; + +export type ValueGetterFn = ( + item: IDataObject, + field: string, +) => IDataObject | IDataObject[] | GenericValue | GenericValue[]; + +function isEmpty(value: T) { + return value === undefined || value === null || value === ''; +} + +function parseReturnData(returnData: IDataObject) { + const regexBrackets = /[\]\["]/g; + const regexSpaces = /[ .]/g; + for (const key of Object.keys(returnData)) { + if (key.match(regexBrackets)) { + const newKey = key.replace(regexBrackets, ''); + returnData[newKey] = returnData[key]; + delete returnData[key]; + } + } + for (const key of Object.keys(returnData)) { + if (key.match(regexSpaces)) { + const newKey = key.replace(regexSpaces, '_'); + returnData[newKey] = returnData[key]; + delete returnData[key]; + } + } +} + +function parseFieldName(fieldName: string[]) { + const regexBrackets = /[\]\["]/g; + const regexSpaces = /[ .]/g; + fieldName = fieldName.map((field) => { + field = field.replace(regexBrackets, ''); + field = field.replace(regexSpaces, '_'); + return field; + }); + return fieldName; +} + +export const fieldValueGetter = (disableDotNotation?: boolean) => { + if (disableDotNotation) { + return (item: IDataObject, field: string) => item[field]; + } else { + return (item: IDataObject, field: string) => get(item, field); + } +}; + +export function checkIfFieldExists( + this: IExecuteFunctions, + items: IDataObject[], + aggregations: Aggregations, + getValue: ValueGetterFn, +) { + for (const aggregation of aggregations) { + if (aggregation.field === '') { + continue; + } + const exist = items.some((item) => getValue(item, aggregation.field) !== undefined); + if (!exist) { + throw new NodeOperationError( + this.getNode(), + `The field '${aggregation.field}' does not exist in any items`, + ); + } + } +} + +function aggregate(items: IDataObject[], entry: Aggregation, getValue: ValueGetterFn) { + const { aggregation, field } = entry; + let data = [...items]; + + if (NUMERICAL_AGGREGATIONS.includes(aggregation)) { + data = data.filter( + (item) => typeof getValue(item, field) === 'number' && !isEmpty(getValue(item, field)), + ); + } + + switch (aggregation) { + //combine operations + case 'append': + if (!entry.includeEmpty) { + data = data.filter((item) => !isEmpty(getValue(item, field))); + } + return data.map((item) => getValue(item, field)); + case 'concatenate': + const separateBy = entry.separateBy === 'other' ? entry.customSeparator : entry.separateBy; + if (!entry.includeEmpty) { + data = data.filter((item) => !isEmpty(getValue(item, field))); + } + return data + .map((item) => { + let value = getValue(item, field); + if (typeof value === 'object') { + value = JSON.stringify(value); + } + if (typeof value === 'undefined') { + value = 'undefined'; + } + + return value; + }) + .join(separateBy); + + //numerical operations + case 'average': + return ( + data.reduce((acc, item) => { + return acc + (getValue(item, field) as number); + }, 0) / data.length + ); + case 'sum': + return data.reduce((acc, item) => { + return acc + (getValue(item, field) as number); + }, 0); + //comparison operations + case 'min': + let min; + for (const item of data) { + const value = getValue(item, field); + if (value !== undefined && value !== null && value !== '') { + if (min === undefined || value < min) { + min = value; + } + } + } + return min !== undefined ? min : null; + case 'max': + let max; + for (const item of data) { + const value = getValue(item, field); + if (value !== undefined && value !== null && value !== '') { + if (max === undefined || value > max) { + max = value; + } + } + } + return max !== undefined ? max : null; + + //count operations + case 'countUnique': + return new Set(data.map((item) => getValue(item, field)).filter((item) => !isEmpty(item))) + .size; + default: + //count by default + return data.filter((item) => !isEmpty(getValue(item, field))).length; + } +} + +function aggregateData( + data: IDataObject[], + fieldsToSummarize: Aggregations, + options: SummarizeOptions, + getValue: ValueGetterFn, +) { + const returnData = fieldsToSummarize.reduce((acc, aggregation) => { + acc[`${AggregationDisplayNames[aggregation.aggregation]}${aggregation.field}`] = aggregate( + data, + aggregation, + getValue, + ); + return acc; + }, {} as IDataObject); + parseReturnData(returnData); + if (options.outputFormat === 'singleItem') { + parseReturnData(returnData); + return returnData; + } else { + return { ...returnData, pairedItems: data.map((item) => item._itemIndex as number) }; + } +} + +export function splitData( + splitKeys: string[], + data: IDataObject[], + fieldsToSummarize: Aggregations, + options: SummarizeOptions, + getValue: ValueGetterFn, +) { + if (!splitKeys || splitKeys.length === 0) { + return aggregateData(data, fieldsToSummarize, options, getValue); + } + + const [firstSplitKey, ...restSplitKeys] = splitKeys; + + const groupedData = data.reduce((acc, item) => { + let keyValuee = getValue(item, firstSplitKey) as string; + + if (typeof keyValuee === 'object') { + keyValuee = JSON.stringify(keyValuee); + } + + if (options.skipEmptySplitFields && typeof keyValuee !== 'number' && !keyValuee) { + return acc; + } + + if (acc[keyValuee] === undefined) { + acc[keyValuee] = [item]; + } else { + (acc[keyValuee] as IDataObject[]).push(item); + } + return acc; + }, {} as IDataObject); + + return Object.keys(groupedData).reduce((acc, key) => { + const value = groupedData[key] as IDataObject[]; + acc[key] = splitData(restSplitKeys, value, fieldsToSummarize, options, getValue); + return acc; + }, {} as IDataObject); +} + +export function aggregationToArray( + aggregationResult: IDataObject, + fieldsToSplitBy: string[], + previousStage: IDataObject = {}, +) { + const returnData: IDataObject[] = []; + fieldsToSplitBy = parseFieldName(fieldsToSplitBy); + const splitFieldName = fieldsToSplitBy[0]; + const isNext = fieldsToSplitBy[1]; + + if (isNext === undefined) { + for (const fieldName of Object.keys(aggregationResult)) { + returnData.push({ + ...previousStage, + [splitFieldName]: fieldName, + ...(aggregationResult[fieldName] as IDataObject), + }); + } + return returnData; + } else { + for (const key of Object.keys(aggregationResult)) { + returnData.push( + ...aggregationToArray(aggregationResult[key] as IDataObject, fieldsToSplitBy.slice(1), { + ...previousStage, + [splitFieldName]: key, + }), + ); + } + return returnData; + } +} From 27a9e936a13b0098669898405e9cc760a59ea935 Mon Sep 17 00:00:00 2001 From: Elias Meire Date: Fri, 1 Dec 2023 16:14:16 +0100 Subject: [PATCH 05/17] Add new icons --- .../nodes/Transform/Aggregate/aggregate.svg | 27 ++++++++++--------- .../nodes/Transform/Limit/limit.svg | 21 ++++++--------- .../RemoveDuplicates/removeDuplicates.svg | 15 +++-------- .../nodes-base/nodes/Transform/Sort/sort.svg | 17 +++--------- .../nodes/Transform/SplitOut/splitOut.svg | 20 +++++--------- .../nodes/Transform/Summarize/summarize.svg | 26 +++++++++--------- 6 files changed, 49 insertions(+), 77 deletions(-) diff --git a/packages/nodes-base/nodes/Transform/Aggregate/aggregate.svg b/packages/nodes-base/nodes/Transform/Aggregate/aggregate.svg index 56100defcee2d..a55ebf954cf64 100644 --- a/packages/nodes-base/nodes/Transform/Aggregate/aggregate.svg +++ b/packages/nodes-base/nodes/Transform/Aggregate/aggregate.svg @@ -1,13 +1,14 @@ - - - - - - \ No newline at end of file + + + + + + + + + + + + + + diff --git a/packages/nodes-base/nodes/Transform/Limit/limit.svg b/packages/nodes-base/nodes/Transform/Limit/limit.svg index 56100defcee2d..ef9b84010f534 100644 --- a/packages/nodes-base/nodes/Transform/Limit/limit.svg +++ b/packages/nodes-base/nodes/Transform/Limit/limit.svg @@ -1,13 +1,8 @@ - - - - - - \ No newline at end of file + + + + + + + + diff --git a/packages/nodes-base/nodes/Transform/RemoveDuplicates/removeDuplicates.svg b/packages/nodes-base/nodes/Transform/RemoveDuplicates/removeDuplicates.svg index de349daca71ab..ce3b3f44b197d 100644 --- a/packages/nodes-base/nodes/Transform/RemoveDuplicates/removeDuplicates.svg +++ b/packages/nodes-base/nodes/Transform/RemoveDuplicates/removeDuplicates.svg @@ -1,13 +1,4 @@ - - - - - + + + diff --git a/packages/nodes-base/nodes/Transform/Sort/sort.svg b/packages/nodes-base/nodes/Transform/Sort/sort.svg index 56100defcee2d..3b1154a00cdd2 100644 --- a/packages/nodes-base/nodes/Transform/Sort/sort.svg +++ b/packages/nodes-base/nodes/Transform/Sort/sort.svg @@ -1,13 +1,4 @@ - - - - - - \ No newline at end of file + + + + diff --git a/packages/nodes-base/nodes/Transform/SplitOut/splitOut.svg b/packages/nodes-base/nodes/Transform/SplitOut/splitOut.svg index 56100defcee2d..34437cd3360d2 100644 --- a/packages/nodes-base/nodes/Transform/SplitOut/splitOut.svg +++ b/packages/nodes-base/nodes/Transform/SplitOut/splitOut.svg @@ -1,13 +1,7 @@ - - - - - - \ No newline at end of file + + + + + + + diff --git a/packages/nodes-base/nodes/Transform/Summarize/summarize.svg b/packages/nodes-base/nodes/Transform/Summarize/summarize.svg index 56100defcee2d..e6db4919f392a 100644 --- a/packages/nodes-base/nodes/Transform/Summarize/summarize.svg +++ b/packages/nodes-base/nodes/Transform/Summarize/summarize.svg @@ -1,13 +1,13 @@ - - - - - - \ No newline at end of file + + + + + + + + + + + + + From be97b05d057b7c0570fe65ef9e918727ba67cea3 Mon Sep 17 00:00:00 2001 From: Elias Meire Date: Mon, 4 Dec 2023 10:02:29 +0100 Subject: [PATCH 06/17] Define sections for subcategories --- .../components/Node/NodeCreator/viewsData.ts | 76 ++++++++++++++++--- packages/editor-ui/src/constants.ts | 10 +++ 2 files changed, 77 insertions(+), 9 deletions(-) diff --git a/packages/editor-ui/src/components/Node/NodeCreator/viewsData.ts b/packages/editor-ui/src/components/Node/NodeCreator/viewsData.ts index 466ae9da1940a..1bc408c0500b3 100644 --- a/packages/editor-ui/src/components/Node/NodeCreator/viewsData.ts +++ b/packages/editor-ui/src/components/Node/NodeCreator/viewsData.ts @@ -29,6 +29,23 @@ import { AI_CATEGORY_EMBEDDING, AI_OTHERS_NODE_CREATOR_VIEW, AI_UNCATEGORIZED_CATEGORY, + SET_NODE_TYPE, + CODE_NODE_TYPE, + DATETIME_NODE_TYPE, + FILTER_NODE_TYPE, + REMOVE_DUPLICATES_NODE_TYPE, + SPLIT_OUT_NODE_TYPE, + LIMIT_NODE_TYPE, + SUMMARIZE_NODE_TYPE, + AGGREGATE_NODE_TYPE, + MERGE_NODE_TYPE, + HTML_NODE_TYPE, + MARKDOWN_NODE_TYPE, + XML_NODE_TYPE, + CRYPTO_NODE_TYPE, + IF_NODE_TYPE, + SPLIT_IN_BATCHES_NODE_TYPE, + HTTP_REQUEST_NODE_TYPE, } from '@/constants'; import { useI18n } from '@/composables/useI18n'; import { useNodeTypesStore } from '@/stores/nodeTypes.store'; @@ -342,15 +359,33 @@ export function RegularView(nodes: SimplifiedNodeType[]) { properties: { title: TRANSFORM_DATA_SUBCATEGORY, icon: 'pen', - }, - }, - { - type: 'subcategory', - key: HELPERS_SUBCATEGORY, - category: CORE_NODES_CATEGORY, - properties: { - title: HELPERS_SUBCATEGORY, - icon: 'toolbox', + sections: [ + { + key: 'popular', + title: i18n.baseText('nodeCreator.sectionNames.popular'), + items: [SET_NODE_TYPE, CODE_NODE_TYPE, DATETIME_NODE_TYPE], + }, + { + key: 'addOrRemove', + title: i18n.baseText('nodeCreator.sectionNames.addOrRemove'), + items: [ + FILTER_NODE_TYPE, + REMOVE_DUPLICATES_NODE_TYPE, + SPLIT_OUT_NODE_TYPE, + LIMIT_NODE_TYPE, + ], + }, + { + key: 'combine', + title: i18n.baseText('nodeCreator.sectionNames.combine'), + items: [SUMMARIZE_NODE_TYPE, AGGREGATE_NODE_TYPE, MERGE_NODE_TYPE], + }, + { + key: 'convert', + title: i18n.baseText('nodeCreator.sectionNames.convert'), + items: [HTML_NODE_TYPE, MARKDOWN_NODE_TYPE, XML_NODE_TYPE, CRYPTO_NODE_TYPE], + }, + ], }, }, { @@ -360,6 +395,13 @@ export function RegularView(nodes: SimplifiedNodeType[]) { properties: { title: FLOWS_CONTROL_SUBCATEGORY, icon: 'code-branch', + sections: [ + { + key: 'popular', + title: i18n.baseText('nodeCreator.sectionNames.popular'), + items: [FILTER_NODE_TYPE, IF_NODE_TYPE, SPLIT_IN_BATCHES_NODE_TYPE, MERGE_NODE_TYPE], + }, + ], }, }, { @@ -371,6 +413,22 @@ export function RegularView(nodes: SimplifiedNodeType[]) { icon: 'file-alt', }, }, + { + type: 'subcategory', + key: ADVANCED_SUBCATEGORY, + category: CORE_NODES_CATEGORY, + properties: { + title: ADVANCED_SUBCATEGORY, + icon: 'toolbox', + sections: [ + { + key: 'popular', + title: i18n.baseText('nodeCreator.sectionNames.popular'), + items: [HTTP_REQUEST_NODE_TYPE, WEBHOOK_NODE_TYPE, CODE_NODE_TYPE], + }, + ], + }, + }, ], }; diff --git a/packages/editor-ui/src/constants.ts b/packages/editor-ui/src/constants.ts index a82d67fb595c2..78aa942f5da8d 100644 --- a/packages/editor-ui/src/constants.ts +++ b/packages/editor-ui/src/constants.ts @@ -157,6 +157,16 @@ export const XERO_NODE_TYPE = 'n8n-nodes-base.xero'; export const ZENDESK_NODE_TYPE = 'n8n-nodes-base.zendesk'; export const ZENDESK_TRIGGER_NODE_TYPE = 'n8n-nodes-base.zendeskTrigger'; export const DISCORD_NODE_TYPE = 'n8n-nodes-base.discord'; +export const DATETIME_NODE_TYPE = 'n8n-nodes-base.dateTime'; +export const REMOVE_DUPLICATES_NODE_TYPE = 'n8n-nodes-base.removeDuplicates'; +export const SPLIT_OUT_NODE_TYPE = 'n8n-nodes-base.splitOut'; +export const LIMIT_NODE_TYPE = 'n8n-nodes-base.limit'; +export const SUMMARIZE_NODE_TYPE = 'n8n-nodes-base.summarize'; +export const AGGREGATE_NODE_TYPE = 'n8n-nodes-base.aggregate'; +export const MERGE_NODE_TYPE = 'n8n-nodes-base.merge'; +export const MARKDOWN_NODE_TYPE = 'n8n-nodes-base.markdown'; +export const XML_NODE_TYPE = 'n8n-nodes-base.xml'; +export const CRYPTO_NODE_TYPE = 'n8n-nodes-base.crypto'; export const CREDENTIAL_ONLY_NODE_PREFIX = 'n8n-creds-base'; export const CREDENTIAL_ONLY_HTTP_NODE_VERSION = 4.1; From cdd8dd3c51fa8f9cbc59cd333c0ad51954f02c69 Mon Sep 17 00:00:00 2001 From: Elias Meire Date: Mon, 4 Dec 2023 14:35:01 +0100 Subject: [PATCH 07/17] Add tests for new list nodes --- .../Transform/Aggregate/Aggregate.node.json | 2 +- .../Aggregate/test/Aggregate.test.ts | 5 + .../Aggregate/test/workflow.aggregate.json | 207 ++++++++++ .../nodes/Transform/Limit/Limit.node.json | 2 +- .../nodes/Transform/Limit/test/Limit.test.ts | 5 + .../Transform/Limit/test/workflow.limit.json | 97 +++++ .../RemoveDuplicates.node.json | 2 +- .../test/RemoveDuplicates.test.ts | 5 + .../test/workflow.removeDuplicates.json | 326 +++++++++++++++ .../nodes/Transform/Sort/Sort.node.json | 2 +- .../nodes/Transform/Sort/Sort.node.ts | 130 +++++- .../nodes/Transform/Sort/test/Sort.test.ts | 5 + .../Transform/Sort/test/workflow.sort.json | 213 ++++++++++ .../Transform/SplitOut/SplitOut.node.json | 2 +- .../Transform/SplitOut/test/SplitOut.test.ts | 5 + .../SplitOut/test/workflow.splitOut.json | 360 ++++++++++++++++ .../test/workflow.splitOutObject.json | 386 ++++++++++++++++++ .../Transform/Summarize/Summarize.node.json | 2 +- .../Summarize/test/Summarize.test.ts | 5 + .../Summarize/test/workflow.summarize.json | 296 ++++++++++++++ 20 files changed, 2050 insertions(+), 7 deletions(-) create mode 100644 packages/nodes-base/nodes/Transform/Aggregate/test/Aggregate.test.ts create mode 100644 packages/nodes-base/nodes/Transform/Aggregate/test/workflow.aggregate.json create mode 100644 packages/nodes-base/nodes/Transform/Limit/test/Limit.test.ts create mode 100644 packages/nodes-base/nodes/Transform/Limit/test/workflow.limit.json create mode 100644 packages/nodes-base/nodes/Transform/RemoveDuplicates/test/RemoveDuplicates.test.ts create mode 100644 packages/nodes-base/nodes/Transform/RemoveDuplicates/test/workflow.removeDuplicates.json create mode 100644 packages/nodes-base/nodes/Transform/Sort/test/Sort.test.ts create mode 100644 packages/nodes-base/nodes/Transform/Sort/test/workflow.sort.json create mode 100644 packages/nodes-base/nodes/Transform/SplitOut/test/SplitOut.test.ts create mode 100644 packages/nodes-base/nodes/Transform/SplitOut/test/workflow.splitOut.json create mode 100644 packages/nodes-base/nodes/Transform/SplitOut/test/workflow.splitOutObject.json create mode 100644 packages/nodes-base/nodes/Transform/Summarize/test/Summarize.test.ts create mode 100644 packages/nodes-base/nodes/Transform/Summarize/test/workflow.summarize.json diff --git a/packages/nodes-base/nodes/Transform/Aggregate/Aggregate.node.json b/packages/nodes-base/nodes/Transform/Aggregate/Aggregate.node.json index 7cf41cfed2daa..53a9760c16858 100644 --- a/packages/nodes-base/nodes/Transform/Aggregate/Aggregate.node.json +++ b/packages/nodes-base/nodes/Transform/Aggregate/Aggregate.node.json @@ -14,6 +14,6 @@ }, "alias": ["Aggregate", "Combine", "Flatten", "Transform", "Array", "List", "Item"], "subcategories": { - "Core Nodes": ["Helpers", "Data Transformation"] + "Core Nodes": ["Data Transformation"] } } diff --git a/packages/nodes-base/nodes/Transform/Aggregate/test/Aggregate.test.ts b/packages/nodes-base/nodes/Transform/Aggregate/test/Aggregate.test.ts new file mode 100644 index 0000000000000..9682e72a11e63 --- /dev/null +++ b/packages/nodes-base/nodes/Transform/Aggregate/test/Aggregate.test.ts @@ -0,0 +1,5 @@ +import { testWorkflows, getWorkflowFilenames } from '@test/nodes/Helpers'; + +const workflows = getWorkflowFilenames(__dirname); + +describe('Test Aggregate Node', () => testWorkflows(workflows)); diff --git a/packages/nodes-base/nodes/Transform/Aggregate/test/workflow.aggregate.json b/packages/nodes-base/nodes/Transform/Aggregate/test/workflow.aggregate.json new file mode 100644 index 0000000000000..db8e778e9731a --- /dev/null +++ b/packages/nodes-base/nodes/Transform/Aggregate/test/workflow.aggregate.json @@ -0,0 +1,207 @@ +{ + "name": "itemLists test", + "nodes": [ + { + "parameters": {}, + "id": "6c90bf81-0c0e-4c5f-9f0c-297f06d9668a", + "name": "When clicking \"Execute Workflow\"", + "type": "n8n-nodes-base.manualTrigger", + "typeVersion": 1, + "position": [-440, 260] + }, + { + "parameters": { + "jsCode": "return [\n {id: 1, char: 'a'},\n {id: 2, char: 'b'},\n {id: 3, char: 'c'},\n {id: 4, char: 'd'},\n {id: 5, char: 'e'},\n];" + }, + "id": "2e0011d5-c6a0-4a40-ab8c-9d011cde40d5", + "name": "Code", + "type": "n8n-nodes-base.code", + "typeVersion": 1, + "position": [-180, 260] + }, + { + "parameters": { + "fieldsToAggregate": { + "fieldToAggregate": [ + { + "fieldToAggregate": "id", + "renameField": true, + "outputFieldName": "data" + } + ] + }, + "options": {} + }, + "id": "d95ca3a3-fb43-4037-846e-b87103dec1a3", + "name": "fields aggregate and rename", + "type": "n8n-nodes-base.aggregate", + "typeVersion": 1, + "position": [80, 0] + }, + { + "parameters": { + "aggregate": "aggregateAllItemData" + }, + "id": "4c1bc7be-7611-418d-aad5-8642b1cc0781", + "name": "aggregate all fields into list", + "type": "n8n-nodes-base.aggregate", + "typeVersion": 1, + "position": [80, 320] + }, + { + "parameters": { + "aggregate": "aggregateAllItemData", + "include": "specifiedFields", + "fieldsToInclude": ["id"] + }, + "id": "951de23c-2018-437b-961e-8ae7d7fd1a82", + "name": "aggregate selected fields into list", + "type": "n8n-nodes-base.aggregate", + "typeVersion": 1, + "position": [80, 500] + }, + { + "parameters": { + "aggregate": "aggregateAllItemData", + "destinationFieldName": "output", + "include": "allFieldsExcept", + "fieldsToExclude": ["char"] + }, + "id": "b62c02ee-5edb-473d-a755-7fb8700641fa", + "name": "aggregate all fields except selected into list", + "type": "n8n-nodes-base.aggregate", + "typeVersion": 1, + "position": [80, 700] + } + ], + "pinData": { + "fields aggregate and rename": [ + { + "json": { + "data": [1, 2, 3, 4, 5] + } + } + ], + "aggregate all fields into list": [ + { + "json": { + "data": [ + { + "id": 1, + "char": "a" + }, + { + "id": 2, + "char": "b" + }, + { + "id": 3, + "char": "c" + }, + { + "id": 4, + "char": "d" + }, + { + "id": 5, + "char": "e" + } + ] + } + } + ], + "aggregate selected fields into list": [ + { + "json": { + "data": [ + { + "id": 1 + }, + { + "id": 2 + }, + { + "id": 3 + }, + { + "id": 4 + }, + { + "id": 5 + } + ] + } + } + ], + "aggregate all fields except selected into list": [ + { + "json": { + "output": [ + { + "id": 1 + }, + { + "id": 2 + }, + { + "id": 3 + }, + { + "id": 4 + }, + { + "id": 5 + } + ] + } + } + ] + }, + "connections": { + "When clicking \"Execute Workflow\"": { + "main": [ + [ + { + "node": "Code", + "type": "main", + "index": 0 + } + ] + ] + }, + "Code": { + "main": [ + [ + { + "node": "fields aggregate and rename", + "type": "main", + "index": 0 + }, + { + "node": "aggregate all fields into list", + "type": "main", + "index": 0 + }, + { + "node": "aggregate selected fields into list", + "type": "main", + "index": 0 + }, + { + "node": "aggregate all fields except selected into list", + "type": "main", + "index": 0 + } + ] + ] + } + }, + "active": false, + "settings": {}, + "versionId": "9bf7c52b-b118-4dad-bfef-7db41828393b", + "id": "105", + "meta": { + "instanceId": "36203ea1ce3cef713fa25999bd9874ae26b9e4c2c3a90a365f2882a154d031d0" + }, + "tags": [] +} diff --git a/packages/nodes-base/nodes/Transform/Limit/Limit.node.json b/packages/nodes-base/nodes/Transform/Limit/Limit.node.json index 68faf8da85f01..1fa3619523a44 100644 --- a/packages/nodes-base/nodes/Transform/Limit/Limit.node.json +++ b/packages/nodes-base/nodes/Transform/Limit/Limit.node.json @@ -14,6 +14,6 @@ }, "alias": ["Limit", "Remove", "Slice", "Transform", "Array", "List", "Item"], "subcategories": { - "Core Nodes": ["Helpers", "Data Transformation"] + "Core Nodes": ["Data Transformation"] } } diff --git a/packages/nodes-base/nodes/Transform/Limit/test/Limit.test.ts b/packages/nodes-base/nodes/Transform/Limit/test/Limit.test.ts new file mode 100644 index 0000000000000..1b9feb88cb73f --- /dev/null +++ b/packages/nodes-base/nodes/Transform/Limit/test/Limit.test.ts @@ -0,0 +1,5 @@ +import { testWorkflows, getWorkflowFilenames } from '@test/nodes/Helpers'; + +const workflows = getWorkflowFilenames(__dirname); + +describe('Test Limit Node', () => testWorkflows(workflows)); diff --git a/packages/nodes-base/nodes/Transform/Limit/test/workflow.limit.json b/packages/nodes-base/nodes/Transform/Limit/test/workflow.limit.json new file mode 100644 index 0000000000000..609538fceb3c0 --- /dev/null +++ b/packages/nodes-base/nodes/Transform/Limit/test/workflow.limit.json @@ -0,0 +1,97 @@ +{ + "name": "itemLists test", + "nodes": [ + { + "parameters": {}, + "id": "bd7af0bb-de39-44b4-ac11-eb1d22f5e8d7", + "name": "When clicking \"Execute Workflow\"", + "type": "n8n-nodes-base.manualTrigger", + "typeVersion": 1, + "position": [260, 180] + }, + { + "parameters": { + "jsCode": "return [\n {entry: 1},\n {entry: 2},\n {entry: 3},\n {entry: 4},\n {entry: 5},\n];" + }, + "id": "21185d7a-f0c1-49a0-9c2d-f0f198ceea7e", + "name": "Code", + "type": "n8n-nodes-base.code", + "typeVersion": 1, + "position": [520, 180] + }, + { + "parameters": { + "maxItems": 1 + }, + "id": "7cc02cc4-1f5f-489a-81e2-4c96b3bdf221", + "name": "Item Lists limit first", + "type": "n8n-nodes-base.limit", + "typeVersion": 1, + "position": [740, 80] + }, + { + "parameters": { + "keep": "lastItems", + "maxItems": 1 + }, + "id": "2bf79d53-7a0b-4716-aa09-55ad43d306ae", + "name": "Item Lists limit last", + "type": "n8n-nodes-base.limit", + "typeVersion": 1, + "position": [740, 300] + } + ], + "pinData": { + "Item Lists limit first": [ + { + "json": { + "entry": 1 + } + } + ], + "Item Lists limit last": [ + { + "json": { + "entry": 5 + } + } + ] + }, + "connections": { + "When clicking \"Execute Workflow\"": { + "main": [ + [ + { + "node": "Code", + "type": "main", + "index": 0 + } + ] + ] + }, + "Code": { + "main": [ + [ + { + "node": "Item Lists limit first", + "type": "main", + "index": 0 + }, + { + "node": "Item Lists limit last", + "type": "main", + "index": 0 + } + ] + ] + } + }, + "active": false, + "settings": {}, + "versionId": "5036d554-1ba4-4b5f-ba9f-1de6df09e807", + "id": "105", + "meta": { + "instanceId": "36203ea1ce3cef713fa25999bd9874ae26b9e4c2c3a90a365f2882a154d031d0" + }, + "tags": [] +} diff --git a/packages/nodes-base/nodes/Transform/RemoveDuplicates/RemoveDuplicates.node.json b/packages/nodes-base/nodes/Transform/RemoveDuplicates/RemoveDuplicates.node.json index 5db54477b4c80..7dbdfc40865e4 100644 --- a/packages/nodes-base/nodes/Transform/RemoveDuplicates/RemoveDuplicates.node.json +++ b/packages/nodes-base/nodes/Transform/RemoveDuplicates/RemoveDuplicates.node.json @@ -24,6 +24,6 @@ "Item" ], "subcategories": { - "Core Nodes": ["Helpers", "Data Transformation"] + "Core Nodes": ["Data Transformation"] } } diff --git a/packages/nodes-base/nodes/Transform/RemoveDuplicates/test/RemoveDuplicates.test.ts b/packages/nodes-base/nodes/Transform/RemoveDuplicates/test/RemoveDuplicates.test.ts new file mode 100644 index 0000000000000..73d697bfc7a06 --- /dev/null +++ b/packages/nodes-base/nodes/Transform/RemoveDuplicates/test/RemoveDuplicates.test.ts @@ -0,0 +1,5 @@ +import { testWorkflows, getWorkflowFilenames } from '@test/nodes/Helpers'; + +const workflows = getWorkflowFilenames(__dirname); + +describe('Test Remove Duplicates Node', () => testWorkflows(workflows)); diff --git a/packages/nodes-base/nodes/Transform/RemoveDuplicates/test/workflow.removeDuplicates.json b/packages/nodes-base/nodes/Transform/RemoveDuplicates/test/workflow.removeDuplicates.json new file mode 100644 index 0000000000000..0699c46f7d451 --- /dev/null +++ b/packages/nodes-base/nodes/Transform/RemoveDuplicates/test/workflow.removeDuplicates.json @@ -0,0 +1,326 @@ +{ + "name": "Remove Duplicates", + "nodes": [ + { + "parameters": {}, + "id": "a4da10da-991f-48ab-b873-9d633a11311f", + "name": "When clicking \"Execute Workflow\"", + "type": "n8n-nodes-base.manualTrigger", + "typeVersion": 1, + "position": [ + 760, + 420 + ] + }, + { + "parameters": { + "jsCode": "return [{ id: 1, name: 'John Doe', age: 18 },{ id: 1, name: 'John Doe', age: 18 },\n { id: 1, name: 'John Doe', age: 98 },\n { id: 3, name: 'Bob Johnson', age:34 }]" + }, + "id": "7ab7d5cd-0b1e-48bc-bdbd-57c91e201cf3", + "name": "Code", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + 980, + 420 + ] + }, + { + "parameters": {}, + "id": "c336939c-062e-475e-ba7c-8601e3662e8c", + "name": "Remove Duplicates (All Fields)", + "type": "n8n-nodes-base.removeDuplicates", + "typeVersion": 1, + "position": [ + 1200, + 260 + ] + }, + { + "parameters": { + "compare": "selectedFields", + "fieldsToCompare": "name", + "options": {} + }, + "id": "d4343ffe-8a9e-4e34-a0a1-aa463afedd80", + "name": "Remove Duplicates (Selected Fields)", + "type": "n8n-nodes-base.removeDuplicates", + "typeVersion": 1, + "position": [ + 1200, + 420 + ] + }, + { + "parameters": { + "compare": "allFieldsExcept", + "fieldsToExclude": "age", + "options": {} + }, + "id": "b67daea4-4545-429e-9e2a-58f2d6a7df7b", + "name": "Remove Duplicates (Except Fields)", + "type": "n8n-nodes-base.removeDuplicates", + "typeVersion": 1, + "position": [ + 1200, + 580 + ] + }, + { + "parameters": {}, + "id": "813e690f-a83e-4a38-a64a-c3d72afcc9ba", + "name": "All Fields", + "type": "n8n-nodes-base.noOp", + "typeVersion": 1, + "position": [ + 1380, + 260 + ] + }, + { + "parameters": {}, + "id": "b5c5c946-2e96-451b-b9a6-78e478504d6c", + "name": "Selected Fields", + "type": "n8n-nodes-base.noOp", + "typeVersion": 1, + "position": [ + 1380, + 420 + ] + }, + { + "parameters": {}, + "id": "afb92bc5-beba-4b0a-aefb-b47cc708a125", + "name": "Except Fields", + "type": "n8n-nodes-base.noOp", + "typeVersion": 1, + "position": [ + 1380, + 580 + ] + }, + { + "parameters": { + "compare": "allFieldsExcept", + "fieldsToExclude": "age", + "options": { + "removeOtherFields": true + } + }, + "id": "f92c5533-ac29-476c-aebb-4849ddd22110", + "name": "Remove Duplicates (Remove)", + "type": "n8n-nodes-base.removeDuplicates", + "typeVersion": 1, + "position": [ + 1200, + 760 + ] + }, + { + "parameters": {}, + "id": "1e142ab7-b32e-4f67-b5cc-5c9fb63fba89", + "name": "Remove", + "type": "n8n-nodes-base.noOp", + "typeVersion": 1, + "position": [ + 1380, + 760 + ] + } + ], + "pinData": { + "Code": [ + { + "json": { + "id": 1, + "name": "John Doe", + "age": 18 + } + }, + { + "json": { + "id": 1, + "name": "John Doe", + "age": 18 + } + }, + { + "json": { + "id": 1, + "name": "John Doe", + "age": 98 + } + }, + { + "json": { + "id": 3, + "name": "Bob Johnson", + "age": 34 + } + } + ], + "All Fields": [ + { + "json": { + "id": 1, + "name": "John Doe", + "age": 18 + } + }, + { + "json": { + "id": 1, + "name": "John Doe", + "age": 98 + } + }, + { + "json": { + "id": 3, + "name": "Bob Johnson", + "age": 34 + } + } + ], + "Selected Fields": [ + { + "json": { + "id": 1, + "name": "John Doe", + "age": 18 + } + }, + { + "json": { + "id": 3, + "name": "Bob Johnson", + "age": 34 + } + } + ], + "Except Fields": [ + { + "json": { + "id": 1, + "name": "John Doe", + "age": 18 + } + }, + { + "json": { + "id": 3, + "name": "Bob Johnson", + "age": 34 + } + } + ], + "Remove": [ + { + "json": { + "id": 1, + "name": "John Doe" + } + }, + { + "json": { + "id": 3, + "name": "Bob Johnson" + } + } + ] + }, + "connections": { + "When clicking \"Execute Workflow\"": { + "main": [ + [ + { + "node": "Code", + "type": "main", + "index": 0 + } + ] + ] + }, + "Code": { + "main": [ + [ + { + "node": "Remove Duplicates (All Fields)", + "type": "main", + "index": 0 + }, + { + "node": "Remove Duplicates (Selected Fields)", + "type": "main", + "index": 0 + }, + { + "node": "Remove Duplicates (Except Fields)", + "type": "main", + "index": 0 + }, + { + "node": "Remove Duplicates (Remove)", + "type": "main", + "index": 0 + } + ] + ] + }, + "Remove Duplicates (All Fields)": { + "main": [ + [ + { + "node": "All Fields", + "type": "main", + "index": 0 + } + ] + ] + }, + "Remove Duplicates (Selected Fields)": { + "main": [ + [ + { + "node": "Selected Fields", + "type": "main", + "index": 0 + } + ] + ] + }, + "Remove Duplicates (Except Fields)": { + "main": [ + [ + { + "node": "Except Fields", + "type": "main", + "index": 0 + } + ] + ] + }, + "Remove Duplicates (Remove)": { + "main": [ + [ + { + "node": "Remove", + "type": "main", + "index": 0 + } + ] + ] + } + }, + "active": false, + "settings": { + "executionOrder": "v1" + }, + "versionId": "5bb09766-4c67-4fb4-ae53-89d8db4727e3", + "id": "74gMYOHjjPArZg4q", + "meta": { + "instanceId": "27cc9b56542ad45b38725555722c50a1c3fee1670bbb67980558314ee08517c4" + }, + "tags": [ + ] + } diff --git a/packages/nodes-base/nodes/Transform/Sort/Sort.node.json b/packages/nodes-base/nodes/Transform/Sort/Sort.node.json index 88a714a7636a8..13e63540d2afb 100644 --- a/packages/nodes-base/nodes/Transform/Sort/Sort.node.json +++ b/packages/nodes-base/nodes/Transform/Sort/Sort.node.json @@ -14,6 +14,6 @@ }, "alias": ["Sort", "Order", "Transform", "Array", "List", "Item"], "subcategories": { - "Core Nodes": ["Helpers", "Data Transformation"] + "Core Nodes": ["Data Transformation"] } } diff --git a/packages/nodes-base/nodes/Transform/Sort/Sort.node.ts b/packages/nodes-base/nodes/Transform/Sort/Sort.node.ts index 67f131eebfe9b..acfa6a7d45d59 100644 --- a/packages/nodes-base/nodes/Transform/Sort/Sort.node.ts +++ b/packages/nodes-base/nodes/Transform/Sort/Sort.node.ts @@ -25,7 +25,135 @@ export class Sort implements INodeType { }, inputs: ['main'], outputs: ['main'], - properties: [], + properties: [ + { + displayName: 'Type', + name: 'type', + type: 'options', + options: [ + { + name: 'Simple', + value: 'simple', + }, + { + name: 'Random', + value: 'random', + }, + { + name: 'Code', + value: 'code', + }, + ], + default: 'simple', + description: 'The fields of the input items to compare to see if they are the same', + }, + { + displayName: 'Fields To Sort By', + name: 'sortFieldsUi', + type: 'fixedCollection', + typeOptions: { + multipleValues: true, + }, + placeholder: 'Add Field To Sort By', + options: [ + { + displayName: '', + name: 'sortField', + values: [ + { + displayName: 'Field Name', + name: 'fieldName', + type: 'string', + required: true, + default: '', + description: 'The field to sort by', + // eslint-disable-next-line n8n-nodes-base/node-param-placeholder-miscased-id + placeholder: 'e.g. id', + hint: ' Enter the field name as text', + requiresDataPath: 'single', + }, + { + displayName: 'Order', + name: 'order', + type: 'options', + options: [ + { + name: 'Ascending', + value: 'ascending', + }, + { + name: 'Descending', + value: 'descending', + }, + ], + default: 'ascending', + description: 'The order to sort by', + }, + ], + }, + ], + default: {}, + description: 'The fields of the input items to compare to see if they are the same', + displayOptions: { + show: { + type: ['simple'], + }, + }, + }, + { + displayName: 'Code', + name: 'code', + type: 'string', + typeOptions: { + alwaysOpenEditWindow: true, + editor: 'code', + rows: 10, + }, + default: `// The two items to compare are in the variables a and b + // Access the fields in a.json and b.json + // Return -1 if a should go before b + // Return 1 if b should go before a + // Return 0 if there's no difference + + fieldName = 'myField'; + + if (a.json[fieldName] < b.json[fieldName]) { + return -1; + } + if (a.json[fieldName] > b.json[fieldName]) { + return 1; + } + return 0;`, + description: 'Javascript code to determine the order of any two items', + displayOptions: { + show: { + type: ['code'], + }, + }, + }, + { + displayName: 'Options', + name: 'options', + type: 'collection', + placeholder: 'Add Field', + default: {}, + displayOptions: { + show: { + type: ['simple'], + }, + }, + options: [ + { + displayName: 'Disable Dot Notation', + name: 'disableDotNotation', + type: 'boolean', + default: false, + description: + 'Whether to disallow referencing child fields using `parent.child` in the field name', + }, + ], + }, + ], }; async execute(this: IExecuteFunctions): Promise { diff --git a/packages/nodes-base/nodes/Transform/Sort/test/Sort.test.ts b/packages/nodes-base/nodes/Transform/Sort/test/Sort.test.ts new file mode 100644 index 0000000000000..fdbe316072e9e --- /dev/null +++ b/packages/nodes-base/nodes/Transform/Sort/test/Sort.test.ts @@ -0,0 +1,5 @@ +import { testWorkflows, getWorkflowFilenames } from '@test/nodes/Helpers'; + +const workflows = getWorkflowFilenames(__dirname); + +describe('Test Sort Node', () => testWorkflows(workflows)); diff --git a/packages/nodes-base/nodes/Transform/Sort/test/workflow.sort.json b/packages/nodes-base/nodes/Transform/Sort/test/workflow.sort.json new file mode 100644 index 0000000000000..b6f6dd53419db --- /dev/null +++ b/packages/nodes-base/nodes/Transform/Sort/test/workflow.sort.json @@ -0,0 +1,213 @@ +{ + "name": "sort test", + "nodes": [ + { + "parameters": {}, + "id": "6c90bf81-0c0e-4c5f-9f0c-297f06d9668a", + "name": "When clicking \"Execute Workflow\"", + "type": "n8n-nodes-base.manualTrigger", + "typeVersion": 1, + "position": [-440, 300] + }, + { + "parameters": { + "jsCode": "return [\n {id: 3, char: 'c'},\n {id: 4, char: 'd'},\n {id: 5, char: 'e'},\n {id: 1, char: 'a'},\n {id: 2, char: 'b'},\n];" + }, + "id": "2e0011d5-c6a0-4a40-ab8c-9d011cde40d5", + "name": "Code", + "type": "n8n-nodes-base.code", + "typeVersion": 1, + "position": [-180, 300] + }, + { + "parameters": { + "sortFieldsUi": { + "sortField": [ + { + "fieldName": "char", + "order": "descending" + }, + { + "fieldName": "id", + "order": "descending" + } + ] + }, + "options": {} + }, + "id": "20031848-2374-45b2-98db-69d7b8d055ad", + "name": "Item Lists1", + "type": "n8n-nodes-base.sort", + "typeVersion": 1, + "position": [80, 300] + }, + { + "parameters": { + "sortFieldsUi": { + "sortField": [ + { + "fieldName": "id" + } + ] + }, + "options": {} + }, + "id": "93dd8c32-21e1-4762-a340-b3e8c6866811", + "name": "Item Lists", + "type": "n8n-nodes-base.sort", + "typeVersion": 1, + "position": [80, 120] + }, + { + "parameters": { + "type": "code", + "code": "// The two items to compare are in the variables a and b\n// Access the fields in a.json and b.json\n// Return -1 if a should go before b\n// Return 1 if b should go before a\n// Return 0 if there's no difference\n\nfieldName = 'id';\n\nif (a.json[fieldName] < b.json[fieldName]) {\n\t\treturn -1;\n}\nif (a.json[fieldName] > b.json[fieldName]) {\n\t\treturn 1;\n}\nreturn 0;" + }, + "id": "112c72e6-b5d9-4d6d-87fc-2621fbaa5bf7", + "name": "Item Lists2", + "type": "n8n-nodes-base.sort", + "typeVersion": 1, + "position": [80, 500] + } + ], + "pinData": { + "Item Lists": [ + { + "json": { + "id": 1, + "char": "a" + } + }, + { + "json": { + "id": 2, + "char": "b" + } + }, + { + "json": { + "id": 3, + "char": "c" + } + }, + { + "json": { + "id": 4, + "char": "d" + } + }, + { + "json": { + "id": 5, + "char": "e" + } + } + ], + "Item Lists1": [ + { + "json": { + "id": 5, + "char": "e" + } + }, + { + "json": { + "id": 4, + "char": "d" + } + }, + { + "json": { + "id": 3, + "char": "c" + } + }, + { + "json": { + "id": 2, + "char": "b" + } + }, + { + "json": { + "id": 1, + "char": "a" + } + } + ], + "Item Lists2": [ + { + "json": { + "id": 1, + "char": "a" + } + }, + { + "json": { + "id": 2, + "char": "b" + } + }, + { + "json": { + "id": 3, + "char": "c" + } + }, + { + "json": { + "id": 4, + "char": "d" + } + }, + { + "json": { + "id": 5, + "char": "e" + } + } + ] + }, + "connections": { + "When clicking \"Execute Workflow\"": { + "main": [ + [ + { + "node": "Code", + "type": "main", + "index": 0 + } + ] + ] + }, + "Code": { + "main": [ + [ + { + "node": "Item Lists1", + "type": "main", + "index": 0 + }, + { + "node": "Item Lists", + "type": "main", + "index": 0 + }, + { + "node": "Item Lists2", + "type": "main", + "index": 0 + } + ] + ] + } + }, + "active": false, + "settings": {}, + "versionId": "6f896427-a3be-44bc-898f-c1a6f58fa1e1", + "id": "105", + "meta": { + "instanceId": "36203ea1ce3cef713fa25999bd9874ae26b9e4c2c3a90a365f2882a154d031d0" + }, + "tags": [] +} diff --git a/packages/nodes-base/nodes/Transform/SplitOut/SplitOut.node.json b/packages/nodes-base/nodes/Transform/SplitOut/SplitOut.node.json index 05677d6c8390e..2afc6ec826b0f 100644 --- a/packages/nodes-base/nodes/Transform/SplitOut/SplitOut.node.json +++ b/packages/nodes-base/nodes/Transform/SplitOut/SplitOut.node.json @@ -14,6 +14,6 @@ }, "alias": ["Split", "Nested", "Transform", "Array", "List", "Item"], "subcategories": { - "Core Nodes": ["Helpers", "Data Transformation"] + "Core Nodes": ["Data Transformation"] } } diff --git a/packages/nodes-base/nodes/Transform/SplitOut/test/SplitOut.test.ts b/packages/nodes-base/nodes/Transform/SplitOut/test/SplitOut.test.ts new file mode 100644 index 0000000000000..24c642cfeb59d --- /dev/null +++ b/packages/nodes-base/nodes/Transform/SplitOut/test/SplitOut.test.ts @@ -0,0 +1,5 @@ +import { testWorkflows, getWorkflowFilenames } from '@test/nodes/Helpers'; + +const workflows = getWorkflowFilenames(__dirname); + +describe('Test Split Out Node', () => testWorkflows(workflows)); diff --git a/packages/nodes-base/nodes/Transform/SplitOut/test/workflow.splitOut.json b/packages/nodes-base/nodes/Transform/SplitOut/test/workflow.splitOut.json new file mode 100644 index 0000000000000..9bb7dac8f9fa9 --- /dev/null +++ b/packages/nodes-base/nodes/Transform/SplitOut/test/workflow.splitOut.json @@ -0,0 +1,360 @@ +{ + "name": "splitOut test", + "nodes": [ + { + "parameters": {}, + "id": "6c90bf81-0c0e-4c5f-9f0c-297f06d9668a", + "name": "When clicking \"Execute Workflow\"", + "type": "n8n-nodes-base.manualTrigger", + "typeVersion": 1, + "position": [-400, 400] + }, + { + "parameters": { + "jsCode": "return { \ndata:[\n {id: 3, char: 'c'},\n {id: 4, char: 'd'},\n {id: 5, char: 'e'},\n {id: 1, char: 'a'},\n {id: 2, char: 'b'},\n],\ndata2: [\n {text: 'foo'},\n],\ndata3: [\n {text: 'bar'},\n],\n};" + }, + "id": "2e0011d5-c6a0-4a40-ab8c-9d011cde40d5", + "name": "Code", + "type": "n8n-nodes-base.code", + "typeVersion": 1, + "position": [-180, 400] + }, + { + "parameters": { + "fieldToSplitOut": "data", + "options": {} + }, + "id": "e7eac465-8fe6-498c-9942-ebd47df537c1", + "name": "Item Lists", + "type": "n8n-nodes-base.splitOut", + "typeVersion": 1, + "position": [80, 160] + }, + { + "parameters": { + "fieldToSplitOut": "data", + "include": "allOtherFields", + "options": {} + }, + "id": "09b7fe15-dbad-4ca6-bf1e-3093139d14e5", + "name": "Item Lists1", + "type": "n8n-nodes-base.splitOut", + "typeVersion": 1, + "position": [80, 320] + }, + { + "parameters": { + "fieldToSplitOut": "data", + "include": "selectedOtherFields", + "fieldsToInclude": ["data3"], + + + + + + + "options": {} + }, + "id": "7ea63dc7-8141-4233-af47-9894919c7fe4", + "name": "Item Lists2", + "type": "n8n-nodes-base.splitOut", + "typeVersion": 1, + "position": [80, 480] + }, + { + "parameters": { + "fieldToSplitOut": "data", + "options": { + "destinationFieldName": "output" + } + }, + "id": "89c3c1b4-9577-480a-931f-3b34450b23cb", + "name": "Item Lists3", + "type": "n8n-nodes-base.splitOut", + "typeVersion": 1, + "position": [80, 660] + } + ], + "pinData": { + "Item Lists": [ + { + "json": { + "id": 3, + "char": "c" + } + }, + { + "json": { + "id": 4, + "char": "d" + } + }, + { + "json": { + "id": 5, + "char": "e" + } + }, + { + "json": { + "id": 1, + "char": "a" + } + }, + { + "json": { + "id": 2, + "char": "b" + } + } + ], + "Item Lists1": [ + { + "json": { + "data2": [ + { + "text": "foo" + } + ], + "data3": [ + { + "text": "bar" + } + ], + "data": { + "id": 3, + "char": "c" + } + } + }, + { + "json": { + "data2": [ + { + "text": "foo" + } + ], + "data3": [ + { + "text": "bar" + } + ], + "data": { + "id": 4, + "char": "d" + } + } + }, + { + "json": { + "data2": [ + { + "text": "foo" + } + ], + "data3": [ + { + "text": "bar" + } + ], + "data": { + "id": 5, + "char": "e" + } + } + }, + { + "json": { + "data2": [ + { + "text": "foo" + } + ], + "data3": [ + { + "text": "bar" + } + ], + "data": { + "id": 1, + "char": "a" + } + } + }, + { + "json": { + "data2": [ + { + "text": "foo" + } + ], + "data3": [ + { + "text": "bar" + } + ], + "data": { + "id": 2, + "char": "b" + } + } + } + ], + "Item Lists2": [ + { + "json": { + "data3": [ + { + "text": "bar" + } + ], + "data": { + "id": 3, + "char": "c" + } + } + }, + { + "json": { + "data3": [ + { + "text": "bar" + } + ], + "data": { + "id": 4, + "char": "d" + } + } + }, + { + "json": { + "data3": [ + { + "text": "bar" + } + ], + "data": { + "id": 5, + "char": "e" + } + } + }, + { + "json": { + "data3": [ + { + "text": "bar" + } + ], + "data": { + "id": 1, + "char": "a" + } + } + }, + { + "json": { + "data3": [ + { + "text": "bar" + } + ], + "data": { + "id": 2, + "char": "b" + } + } + } + ], + "Item Lists3": [ + { + "json": { + "output": { + "id": 3, + "char": "c" + } + } + }, + { + "json": { + "output": { + "id": 4, + "char": "d" + } + } + }, + { + "json": { + "output": { + "id": 5, + "char": "e" + } + } + }, + { + "json": { + "output": { + "id": 1, + "char": "a" + } + } + }, + { + "json": { + "output": { + "id": 2, + "char": "b" + } + } + } + ] + }, + "connections": { + "When clicking \"Execute Workflow\"": { + "main": [ + [ + { + "node": "Code", + "type": "main", + "index": 0 + } + ] + ] + }, + "Code": { + "main": [ + [ + { + "node": "Item Lists", + "type": "main", + "index": 0 + }, + { + "node": "Item Lists1", + "type": "main", + "index": 0 + }, + { + "node": "Item Lists2", + "type": "main", + "index": 0 + }, + { + "node": "Item Lists3", + "type": "main", + "index": 0 + } + ] + ] + } + }, + "active": false, + "settings": {}, + "versionId": "9230f580-6f41-47c9-9949-bf258fc3fa47", + "id": "105", + "meta": { + "instanceId": "36203ea1ce3cef713fa25999bd9874ae26b9e4c2c3a90a365f2882a154d031d0" + }, + "tags": [] +} diff --git a/packages/nodes-base/nodes/Transform/SplitOut/test/workflow.splitOutObject.json b/packages/nodes-base/nodes/Transform/SplitOut/test/workflow.splitOutObject.json new file mode 100644 index 0000000000000..dda1c6c4c4f74 --- /dev/null +++ b/packages/nodes-base/nodes/Transform/SplitOut/test/workflow.splitOutObject.json @@ -0,0 +1,386 @@ +{ + "name": "itemList split Object", + "nodes": [ + { + "parameters": {}, + "id": "ade46a75-ab57-48c6-886b-0c118f5ef1c6", + "name": "When clicking \"Execute Workflow\"", + "type": "n8n-nodes-base.manualTrigger", + "typeVersion": 1, + "position": [520, 800] + }, + { + "parameters": { + "fieldToSplitOut": "data", + "include": "selectedOtherFields", + "fieldsToInclude": ["tag"], + "options": {} + }, + "id": "45e1d7a3-d6e8-4b69-a68a-1038db13be4c", + "name": "Item Lists1", + "type": "n8n-nodes-base.splitOut", + "typeVersion": 1, + "position": [1120, 340] + }, + { + "parameters": { + "jsCode": "const data = {\n entry1: {\n id: 1,\n info: 'some info 1',\n },\n entry2: {\n id: 2,\n info: 'some info 2',\n },\n entry3: {\n id: 3,\n info: 'some info 3',\n },\n};\n\n\nconst data2 = [\n 'a', 'b', 'c'\n];\n\nconst data3 = {\n a: 1,\n b: 2,\n c: 3,\n};\n\nreturn {data, data2, data3, data4: null, tag: 'bar'};" + }, + "id": "faa78fac-468d-42b8-96e9-0fb62c312da3", + "name": "Code1", + "type": "n8n-nodes-base.code", + "typeVersion": 1, + "position": [760, 800] + }, + { + "parameters": {}, + "id": "5baaf321-7e89-473d-a313-7cb90b3f13b3", + "name": "No Operation, do nothing", + "type": "n8n-nodes-base.noOp", + "typeVersion": 1, + "position": [1380, 340] + }, + { + "parameters": { + "fieldToSplitOut": "data3", + "include": "allOtherFields", + "options": { + "destinationFieldName": "extracted" + } + }, + "id": "a786bea9-eb29-4c6d-aea6-a22aee622bc6", + "name": "Item Lists", + "type": "n8n-nodes-base.splitOut", + "typeVersion": 1, + "position": [1120, 720] + }, + { + "parameters": {}, + "id": "0521a24b-c74a-48fa-ae50-48a242b97806", + "name": "No Operation, do nothing1", + "type": "n8n-nodes-base.noOp", + "typeVersion": 1, + "position": [1380, 720] + }, + { + "parameters": { + "fieldToSplitOut": "data3", + "options": {} + }, + "id": "0c1c8827-72ab-4738-918c-d529e66505c6", + "name": "Item Lists2", + "type": "n8n-nodes-base.splitOut", + "typeVersion": 1, + "position": [1120, 540] + }, + { + "parameters": {}, + "id": "4c0dca36-c2ae-4d40-8952-0e728ac93fa3", + "name": "No Operation, do nothing2", + "type": "n8n-nodes-base.noOp", + "typeVersion": 1, + "position": [1380, 540] + }, + { + "parameters": { + "fieldToSplitOut": "data2", + "options": {} + }, + "id": "b2031380-b2a8-426d-8f7a-ab072d23b979", + "name": "Item Lists3", + "type": "n8n-nodes-base.splitOut", + "typeVersion": 1, + "position": [1120, 920] + }, + { + "parameters": {}, + "id": "617f7259-beee-42f1-bba2-4e75a83fe369", + "name": "No Operation, do nothing3", + "type": "n8n-nodes-base.noOp", + "typeVersion": 1, + "position": [1380, 920] + }, + { + "parameters": { + "fieldToSplitOut": "data4", + "include": "allOtherFields", + "options": {} + }, + "id": "8909b8eb-e5a9-4436-8e62-09d8c9670ac1", + "name": "Item Lists4", + "type": "n8n-nodes-base.splitOut", + "typeVersion": 1, + "position": [1120, 1140], + "continueOnFail": true + }, + { + "parameters": {}, + "id": "a9278f90-8ad9-42dc-85b6-28bf1b6764b7", + "name": "No Operation, do nothing4", + "type": "n8n-nodes-base.noOp", + "typeVersion": 1, + "position": [1380, 1140] + } + ], + "pinData": { + "No Operation, do nothing": [ + { + "json": { + "tag": "bar", + "data": { + "id": 1, + "info": "some info 1" + } + } + }, + { + "json": { + "tag": "bar", + "data": { + "id": 2, + "info": "some info 2" + } + } + }, + { + "json": { + "tag": "bar", + "data": { + "id": 3, + "info": "some info 3" + } + } + } + ], + "No Operation, do nothing2": [ + { + "json": { + "data3": 1 + } + }, + { + "json": { + "data3": 2 + } + }, + { + "json": { + "data3": 3 + } + } + ], + "No Operation, do nothing1": [ + { + "json": { + "data": { + "entry1": { + "id": 1, + "info": "some info 1" + }, + "entry2": { + "id": 2, + "info": "some info 2" + }, + "entry3": { + "id": 3, + "info": "some info 3" + } + }, + "data2": ["a", "b", "c"], + "data4": null, + "tag": "bar", + "extracted": 1 + } + }, + { + "json": { + "data": { + "entry1": { + "id": 1, + "info": "some info 1" + }, + "entry2": { + "id": 2, + "info": "some info 2" + }, + "entry3": { + "id": 3, + "info": "some info 3" + } + }, + "data2": ["a", "b", "c"], + "data4": null, + "tag": "bar", + "extracted": 2 + } + }, + { + "json": { + "data": { + "entry1": { + "id": 1, + "info": "some info 1" + }, + "entry2": { + "id": 2, + "info": "some info 2" + }, + "entry3": { + "id": 3, + "info": "some info 3" + } + }, + "data2": ["a", "b", "c"], + "data4": null, + "tag": "bar", + "extracted": 3 + } + } + ], + "No Operation, do nothing3": [ + { + "json": { + "data2": "a" + } + }, + { + "json": { + "data2": "b" + } + }, + { + "json": { + "data2": "c" + } + } + ], + "No Operation, do nothing4": [ + { + "json": { + "data": { + "entry1": { + "id": 1, + "info": "some info 1" + }, + "entry2": { + "id": 2, + "info": "some info 2" + }, + "entry3": { + "id": 3, + "info": "some info 3" + } + }, + "data2": ["a", "b", "c"], + "data3": { + "a": 1, + "b": 2, + "c": 3 + }, + "data4": null, + "tag": "bar" + } + } + ] + }, + "connections": { + "When clicking \"Execute Workflow\"": { + "main": [ + [ + { + "node": "Code1", + "type": "main", + "index": 0 + } + ] + ] + }, + "Code1": { + "main": [ + [ + { + "node": "Item Lists1", + "type": "main", + "index": 0 + }, + { + "node": "Item Lists2", + "type": "main", + "index": 0 + }, + { + "node": "Item Lists", + "type": "main", + "index": 0 + }, + { + "node": "Item Lists3", + "type": "main", + "index": 0 + }, + { + "node": "Item Lists4", + "type": "main", + "index": 0 + } + ] + ] + }, + "Item Lists1": { + "main": [ + [ + { + "node": "No Operation, do nothing", + "type": "main", + "index": 0 + } + ] + ] + }, + "Item Lists": { + "main": [ + [ + { + "node": "No Operation, do nothing1", + "type": "main", + "index": 0 + } + ] + ] + }, + "Item Lists2": { + "main": [ + [ + { + "node": "No Operation, do nothing2", + "type": "main", + "index": 0 + } + ] + ] + }, + "Item Lists3": { + "main": [ + [ + { + "node": "No Operation, do nothing3", + "type": "main", + "index": 0 + } + ] + ] + }, + "Item Lists4": { + "main": [ + [ + { + "node": "No Operation, do nothing4", + "type": "main", + "index": 0 + } + ] + ] + } + }, + "active": false +} diff --git a/packages/nodes-base/nodes/Transform/Summarize/Summarize.node.json b/packages/nodes-base/nodes/Transform/Summarize/Summarize.node.json index 6417c4b18b52a..816e6c4d64a15 100644 --- a/packages/nodes-base/nodes/Transform/Summarize/Summarize.node.json +++ b/packages/nodes-base/nodes/Transform/Summarize/Summarize.node.json @@ -27,6 +27,6 @@ "Item" ], "subcategories": { - "Core Nodes": ["Helpers", "Data Transformation"] + "Core Nodes": ["Data Transformation"] } } diff --git a/packages/nodes-base/nodes/Transform/Summarize/test/Summarize.test.ts b/packages/nodes-base/nodes/Transform/Summarize/test/Summarize.test.ts new file mode 100644 index 0000000000000..c846b0c33c9e7 --- /dev/null +++ b/packages/nodes-base/nodes/Transform/Summarize/test/Summarize.test.ts @@ -0,0 +1,5 @@ +import { testWorkflows, getWorkflowFilenames } from '@test/nodes/Helpers'; + +const workflows = getWorkflowFilenames(__dirname); + +describe('Test Summarize Node', () => testWorkflows(workflows)); diff --git a/packages/nodes-base/nodes/Transform/Summarize/test/workflow.summarize.json b/packages/nodes-base/nodes/Transform/Summarize/test/workflow.summarize.json new file mode 100644 index 0000000000000..3a6446d12b9d5 --- /dev/null +++ b/packages/nodes-base/nodes/Transform/Summarize/test/workflow.summarize.json @@ -0,0 +1,296 @@ +{ + "name": "summarize test", + "nodes": [ + { + "parameters": {}, + "id": "6c90bf81-0c0e-4c5f-9f0c-297f06d9668a", + "name": "When clicking \"Execute Workflow\"", + "type": "n8n-nodes-base.manualTrigger", + "typeVersion": 1, + "position": [-400, 420] + }, + { + "parameters": { + "jsCode": "return [\n {\n category: 'red',\n text: 'foo',\n char: 'a',\n value: 1,\n },\n {\n category: 'blue',\n text: 'spam',\n char: 'b',\n value: 2,\n },\n {\n category: 'green',\n text: 'bar',\n char: 'c',\n value: 3,\n },\n {\n category: 'red',\n text: 'foo',\n char: 'a',\n value: 4,\n },\n {\n category: 'red',\n text: 'bar',\n char: 'a',\n value: 5,\n },\n {\n category: 'blue',\n text: 'foo',\n char: 'a',\n value: 6,\n },\n {\n category: 'blue',\n text: 'foo',\n char: 'a',\n value: 7,\n },\n];" + }, + "id": "2e0011d5-c6a0-4a40-ab8c-9d011cde40d5", + "name": "Code", + "type": "n8n-nodes-base.code", + "typeVersion": 1, + "position": [-180, 420] + }, + { + "parameters": { + "fieldsToSummarize": { + "values": [ + { + "aggregation": "append", + "field": "char" + }, + { + "field": "char" + }, + { + "aggregation": "countUnique", + "field": "char" + }, + { + "aggregation": "concatenate", + "field": "char", + "separateBy": ", " + } + ] + }, + "fieldsToSplitBy": "category, text", + "options": {} + }, + "id": "1dedf668-b766-4283-9efd-90db28404f0b", + "name": "Item Lists", + "type": "n8n-nodes-base.summarize", + "typeVersion": 1, + "position": [40, 220] + }, + { + "parameters": { + "fieldsToSummarize": { + "values": [ + { + "aggregation": "append", + "field": "char" + }, + { + "field": "char" + }, + { + "aggregation": "countUnique", + "field": "char" + }, + { + "aggregation": "concatenate", + "field": "char", + "separateBy": ", " + } + ] + }, + "fieldsToSplitBy": "category, text", + "options": { + "outputFormat": "singleItem" + } + }, + "id": "8fd0f819-226c-4b29-87c7-b724dd72605c", + "name": "Item Lists1", + "type": "n8n-nodes-base.summarize", + "typeVersion": 1, + "position": [40, 420] + }, + { + "parameters": { + "fieldsToSummarize": { + "values": [ + { + "aggregation": "average", + "field": "value" + }, + { + "aggregation": "max", + "field": "value" + }, + { + "aggregation": "min", + "field": "value" + }, + { + "aggregation": "max", + "field": "value" + }, + { + "aggregation": "sum", + "field": "value" + }, + { + "aggregation": "append", + "field": "value" + } + ] + }, + "fieldsToSplitBy": "category", + "options": {} + }, + "id": "33e0367d-42d9-4f82-8fc8-8e2019aa3734", + "name": "Item Lists2", + "type": "n8n-nodes-base.summarize", + "typeVersion": 1, + "position": [40, 620] + } + ], + "pinData": { + "Item Lists": [ + { + "json": { + "category": "red", + "text": "foo", + "appended_char": ["a", "a"], + "count_char": 2, + "unique_count_char": 1, + "concatenated_char": "a, a" + } + }, + { + "json": { + "category": "red", + "text": "bar", + "appended_char": ["a"], + "count_char": 1, + "unique_count_char": 1, + "concatenated_char": "a" + } + }, + { + "json": { + "category": "blue", + "text": "spam", + "appended_char": ["b"], + "count_char": 1, + "unique_count_char": 1, + "concatenated_char": "b" + } + }, + { + "json": { + "category": "blue", + "text": "foo", + "appended_char": ["a", "a"], + "count_char": 2, + "unique_count_char": 1, + "concatenated_char": "a, a" + } + }, + { + "json": { + "category": "green", + "text": "bar", + "appended_char": ["c"], + "count_char": 1, + "unique_count_char": 1, + "concatenated_char": "c" + } + } + ], + "Item Lists1": [ + { + "json": { + "red": { + "foo": { + "appended_char": ["a", "a"], + "count_char": 2, + "unique_count_char": 1, + "concatenated_char": "a, a" + }, + "bar": { + "appended_char": ["a"], + "count_char": 1, + "unique_count_char": 1, + "concatenated_char": "a" + } + }, + "blue": { + "spam": { + "appended_char": ["b"], + "count_char": 1, + "unique_count_char": 1, + "concatenated_char": "b" + }, + "foo": { + "appended_char": ["a", "a"], + "count_char": 2, + "unique_count_char": 1, + "concatenated_char": "a, a" + } + }, + "green": { + "bar": { + "appended_char": ["c"], + "count_char": 1, + "unique_count_char": 1, + "concatenated_char": "c" + } + } + } + } + ], + "Item Lists2": [ + { + "json": { + "category": "red", + "average_value": 3.3333333333333335, + "max_value": 5, + "min_value": 1, + "sum_value": 10, + "appended_value": [1, 4, 5] + } + }, + { + "json": { + "category": "blue", + "average_value": 5, + "max_value": 7, + "min_value": 2, + "sum_value": 15, + "appended_value": [2, 6, 7] + } + }, + { + "json": { + "category": "green", + "average_value": 3, + "max_value": 3, + "min_value": 3, + "sum_value": 3, + "appended_value": [3] + } + } + ] + }, + "connections": { + "When clicking \"Execute Workflow\"": { + "main": [ + [ + { + "node": "Code", + "type": "main", + "index": 0 + } + ] + ] + }, + "Code": { + "main": [ + [ + { + "node": "Item Lists", + "type": "main", + "index": 0 + }, + { + "node": "Item Lists1", + "type": "main", + "index": 0 + }, + { + "node": "Item Lists2", + "type": "main", + "index": 0 + } + ] + ] + } + }, + "active": false, + "settings": {}, + "versionId": "bee0d911-844d-4fe6-bd52-a1716dd74dd8", + "id": "105", + "meta": { + "instanceId": "36203ea1ce3cef713fa25999bd9874ae26b9e4c2c3a90a365f2882a154d031d0" + }, + "tags": [] +} From 96691f71f3b4c1d32a19e63810470dab6fdf1ab5 Mon Sep 17 00:00:00 2001 From: Elias Meire Date: Mon, 4 Dec 2023 14:36:13 +0100 Subject: [PATCH 08/17] Add sections for all subcategories except files --- .../Node/NodeCreator/Modes/NodesMode.vue | 2 +- .../src/components/Node/NodeCreator/viewsData.ts | 14 +++++++++----- packages/editor-ui/src/constants.ts | 1 + .../editor-ui/src/plugins/i18n/locales/en.json | 5 ++++- packages/nodes-base/nodes/Code/Code.node.json | 2 +- packages/nodes-base/nodes/Crypto/Crypto.node.json | 2 +- .../nodes-base/nodes/DateTime/DateTime.node.json | 2 +- .../nodes/EmailReadImap/EmailReadImap.node.json | 2 +- .../nodes-base/nodes/EmailSend/EmailSend.node.json | 3 --- .../nodes/ErrorTrigger/ErrorTrigger.node.json | 2 +- .../ExecuteWorkflow/ExecuteWorkflow.node.json | 2 +- packages/nodes-base/nodes/Filter/Filter.node.json | 2 +- .../nodes-base/nodes/Form/FormTrigger.node.json | 2 +- packages/nodes-base/nodes/Html/Html.node.json | 2 +- .../nodes-base/nodes/Markdown/Markdown.node.json | 2 +- packages/nodes-base/nodes/Merge/Merge.node.json | 2 +- packages/nodes-base/nodes/N8n/N8n.node.json | 2 +- .../RespondToWebhook/RespondToWebhook.node.json | 2 +- .../nodes/RssFeedRead/RssFeedRead.node.json | 3 --- .../nodes/Schedule/ScheduleTrigger.node.json | 5 +---- .../nodes/SseTrigger/SseTrigger.node.json | 2 +- .../nodes/StopAndError/StopAndError.node.json | 2 +- packages/nodes-base/nodes/Wait/Wait.node.json | 2 +- .../nodes-base/nodes/Webhook/Webhook.node.json | 2 +- packages/workflow/src/NodeHelpers.ts | 1 + 25 files changed, 34 insertions(+), 34 deletions(-) diff --git a/packages/editor-ui/src/components/Node/NodeCreator/Modes/NodesMode.vue b/packages/editor-ui/src/components/Node/NodeCreator/Modes/NodesMode.vue index d2bcdb2ab9274..4cedb56ca1498 100644 --- a/packages/editor-ui/src/components/Node/NodeCreator/Modes/NodesMode.vue +++ b/packages/editor-ui/src/components/Node/NodeCreator/Modes/NodesMode.vue @@ -162,7 +162,7 @@ function subcategoriesMapper(item: INodeCreateElement) { } function baseSubcategoriesFilter(item: INodeCreateElement): boolean { - if (item.type === 'section') return item.children.every(baseSubcategoriesFilter); + if (item.type === 'section') return true; if (item.type !== 'node') return false; const hasTriggerGroup = item.properties.group.includes('trigger'); diff --git a/packages/editor-ui/src/components/Node/NodeCreator/viewsData.ts b/packages/editor-ui/src/components/Node/NodeCreator/viewsData.ts index ba0e5c02451b5..fa674889f8441 100644 --- a/packages/editor-ui/src/components/Node/NodeCreator/viewsData.ts +++ b/packages/editor-ui/src/components/Node/NodeCreator/viewsData.ts @@ -45,6 +45,9 @@ import { IF_NODE_TYPE, SPLIT_IN_BATCHES_NODE_TYPE, HTTP_REQUEST_NODE_TYPE, + HELPERS_SUBCATEGORY, + RSS_READ_NODE_TYPE, + EMAIL_SEND_NODE_TYPE, } from '@/constants'; import { useI18n } from '@/composables/useI18n'; import { useNodeTypesStore } from '@/stores/nodeTypes.store'; @@ -356,6 +359,7 @@ export function RegularView(nodes: SimplifiedNodeType[]) { properties: { title: 'App Regular Nodes', icon: 'globe', + forceIncludeNodes: [RSS_READ_NODE_TYPE, EMAIL_SEND_NODE_TYPE], }, }, { @@ -373,7 +377,7 @@ export function RegularView(nodes: SimplifiedNodeType[]) { }, { key: 'addOrRemove', - title: i18n.baseText('nodeCreator.sectionNames.addOrRemove'), + title: i18n.baseText('nodeCreator.sectionNames.transform.addOrRemove'), items: [ FILTER_NODE_TYPE, REMOVE_DUPLICATES_NODE_TYPE, @@ -383,12 +387,12 @@ export function RegularView(nodes: SimplifiedNodeType[]) { }, { key: 'combine', - title: i18n.baseText('nodeCreator.sectionNames.combine'), + title: i18n.baseText('nodeCreator.sectionNames.transform.combine'), items: [SUMMARIZE_NODE_TYPE, AGGREGATE_NODE_TYPE, MERGE_NODE_TYPE], }, { key: 'convert', - title: i18n.baseText('nodeCreator.sectionNames.convert'), + title: i18n.baseText('nodeCreator.sectionNames.transform.convert'), items: [HTML_NODE_TYPE, MARKDOWN_NODE_TYPE, XML_NODE_TYPE, CRYPTO_NODE_TYPE], }, ], @@ -421,10 +425,10 @@ export function RegularView(nodes: SimplifiedNodeType[]) { }, { type: 'subcategory', - key: ADVANCED_SUBCATEGORY, + key: HELPERS_SUBCATEGORY, category: CORE_NODES_CATEGORY, properties: { - title: ADVANCED_SUBCATEGORY, + title: HELPERS_SUBCATEGORY, icon: 'toolbox', sections: [ { diff --git a/packages/editor-ui/src/constants.ts b/packages/editor-ui/src/constants.ts index 78aa942f5da8d..ac8b21c0af333 100644 --- a/packages/editor-ui/src/constants.ts +++ b/packages/editor-ui/src/constants.ts @@ -167,6 +167,7 @@ export const MERGE_NODE_TYPE = 'n8n-nodes-base.merge'; export const MARKDOWN_NODE_TYPE = 'n8n-nodes-base.markdown'; export const XML_NODE_TYPE = 'n8n-nodes-base.xml'; export const CRYPTO_NODE_TYPE = 'n8n-nodes-base.crypto'; +export const RSS_READ_NODE_TYPE = 'n8n-nodes-base.rssFeedRead'; export const CREDENTIAL_ONLY_NODE_PREFIX = 'n8n-creds-base'; export const CREDENTIAL_ONLY_HTTP_NODE_VERSION = 4.1; diff --git a/packages/editor-ui/src/plugins/i18n/locales/en.json b/packages/editor-ui/src/plugins/i18n/locales/en.json index 88d92105e23d4..20c8fd82ac39e 100644 --- a/packages/editor-ui/src/plugins/i18n/locales/en.json +++ b/packages/editor-ui/src/plugins/i18n/locales/en.json @@ -884,7 +884,7 @@ "nodeCreator.subcategoryNames.dataTransformation": "Data transformation", "nodeCreator.subcategoryNames.files": "Files", "nodeCreator.subcategoryNames.flow": "Flow", - "nodeCreator.subcategoryNames.helpers": "Helpers", + "nodeCreator.subcategoryNames.helpers": "Advanced", "nodeCreator.subcategoryNames.otherTriggerNodes": "Other ways...", "nodeCreator.subcategoryNames.agents": "Agents", "nodeCreator.subcategoryNames.chains": "Chains", @@ -900,6 +900,9 @@ "nodeCreator.subcategoryNames.miscellaneous": "Miscellaneous", "nodeCreator.sectionNames.popular": "Popular", "nodeCreator.sectionNames.other": "Other", + "nodeCreator.sectionNames.transform.combine": "Combine items", + "nodeCreator.sectionNames.transform.addOrRemove": "Add or remove items", + "nodeCreator.sectionNames.transform.convert": "Convert data", "nodeCreator.triggerHelperPanel.addAnotherTrigger": "Add another trigger", "nodeCreator.triggerHelperPanel.addAnotherTriggerDescription": "Triggers start your workflow. Workflows can have multiple triggers.", "nodeCreator.triggerHelperPanel.title": "When should this workflow run?", diff --git a/packages/nodes-base/nodes/Code/Code.node.json b/packages/nodes-base/nodes/Code/Code.node.json index 8ec1cad6c98f2..57dcb6bc2a268 100644 --- a/packages/nodes-base/nodes/Code/Code.node.json +++ b/packages/nodes-base/nodes/Code/Code.node.json @@ -13,6 +13,6 @@ }, "alias": ["cpde", "Javascript", "JS", "Python", "Script", "Custom Code", "Function"], "subcategories": { - "Core Nodes": ["Data Transformation"] + "Core Nodes": ["Helpers", "Data Transformation"] } } diff --git a/packages/nodes-base/nodes/Crypto/Crypto.node.json b/packages/nodes-base/nodes/Crypto/Crypto.node.json index 16a9b7ea5dd02..30e1febb0862f 100644 --- a/packages/nodes-base/nodes/Crypto/Crypto.node.json +++ b/packages/nodes-base/nodes/Crypto/Crypto.node.json @@ -20,6 +20,6 @@ }, "alias": ["Encrypt", "SHA", "Hash"], "subcategories": { - "Core Nodes": ["Helpers"] + "Core Nodes": ["Data Transformation"] } } diff --git a/packages/nodes-base/nodes/DateTime/DateTime.node.json b/packages/nodes-base/nodes/DateTime/DateTime.node.json index f1d41f281bf5f..8c325f0007af1 100644 --- a/packages/nodes-base/nodes/DateTime/DateTime.node.json +++ b/packages/nodes-base/nodes/DateTime/DateTime.node.json @@ -23,6 +23,6 @@ ] }, "subcategories": { - "Core Nodes": ["Helpers", "Data Transformation"] + "Core Nodes": ["Data Transformation"] } } diff --git a/packages/nodes-base/nodes/EmailReadImap/EmailReadImap.node.json b/packages/nodes-base/nodes/EmailReadImap/EmailReadImap.node.json index 321c472398359..ca29d5e78258f 100644 --- a/packages/nodes-base/nodes/EmailReadImap/EmailReadImap.node.json +++ b/packages/nodes-base/nodes/EmailReadImap/EmailReadImap.node.json @@ -23,6 +23,6 @@ ] }, "subcategories": { - "Core Nodes": ["Helpers", "Other Trigger Nodes"] + "Core Nodes": ["Other Trigger Nodes"] } } diff --git a/packages/nodes-base/nodes/EmailSend/EmailSend.node.json b/packages/nodes-base/nodes/EmailSend/EmailSend.node.json index 51a9454610e5f..3f5a2d5a135b2 100644 --- a/packages/nodes-base/nodes/EmailSend/EmailSend.node.json +++ b/packages/nodes-base/nodes/EmailSend/EmailSend.node.json @@ -27,8 +27,5 @@ } ] }, - "subcategories": { - "Core Nodes": ["Helpers"] - }, "alias": ["SMTP"] } diff --git a/packages/nodes-base/nodes/ErrorTrigger/ErrorTrigger.node.json b/packages/nodes-base/nodes/ErrorTrigger/ErrorTrigger.node.json index 1ea73accf2ab8..63e9c228c18c7 100644 --- a/packages/nodes-base/nodes/ErrorTrigger/ErrorTrigger.node.json +++ b/packages/nodes-base/nodes/ErrorTrigger/ErrorTrigger.node.json @@ -19,6 +19,6 @@ ] }, "subcategories": { - "Core Nodes": ["Helpers", "Other Trigger Nodes"] + "Core Nodes": ["Other Trigger Nodes"] } } diff --git a/packages/nodes-base/nodes/ExecuteWorkflow/ExecuteWorkflow.node.json b/packages/nodes-base/nodes/ExecuteWorkflow/ExecuteWorkflow.node.json index 9bb4ffae1e89a..9fc522970319a 100644 --- a/packages/nodes-base/nodes/ExecuteWorkflow/ExecuteWorkflow.node.json +++ b/packages/nodes-base/nodes/ExecuteWorkflow/ExecuteWorkflow.node.json @@ -13,6 +13,6 @@ }, "alias": ["n8n"], "subcategories": { - "Core Nodes": ["Helpers"] + "Core Nodes": ["Helpers", "Flow"] } } diff --git a/packages/nodes-base/nodes/Filter/Filter.node.json b/packages/nodes-base/nodes/Filter/Filter.node.json index 28a67f27cb57e..2706b44f44662 100644 --- a/packages/nodes-base/nodes/Filter/Filter.node.json +++ b/packages/nodes-base/nodes/Filter/Filter.node.json @@ -14,6 +14,6 @@ }, "alias": ["Router", "Filter", "Condition", "Logic", "Boolean", "Branch"], "subcategories": { - "Core Nodes": ["Flow"] + "Core Nodes": ["Flow", "Data Transformation"] } } diff --git a/packages/nodes-base/nodes/Form/FormTrigger.node.json b/packages/nodes-base/nodes/Form/FormTrigger.node.json index d50582abd9f55..8fb0763a34595 100644 --- a/packages/nodes-base/nodes/Form/FormTrigger.node.json +++ b/packages/nodes-base/nodes/Form/FormTrigger.node.json @@ -13,6 +13,6 @@ "generic": [] }, "subcategories": { - "Core Nodes": ["Helpers", "Other Trigger Nodes"] + "Core Nodes": ["Other Trigger Nodes"] } } diff --git a/packages/nodes-base/nodes/Html/Html.node.json b/packages/nodes-base/nodes/Html/Html.node.json index 6c73ada8bf31e..dbd9b221e4ff1 100644 --- a/packages/nodes-base/nodes/Html/Html.node.json +++ b/packages/nodes-base/nodes/Html/Html.node.json @@ -11,7 +11,7 @@ ] }, "subcategories": { - "Core Nodes": ["Helpers", "Data Transformation"] + "Core Nodes": ["Data Transformation"] }, "alias": ["extract", "template", "table"] } diff --git a/packages/nodes-base/nodes/Markdown/Markdown.node.json b/packages/nodes-base/nodes/Markdown/Markdown.node.json index c4bde73bf051c..1c75d1093f6bd 100644 --- a/packages/nodes-base/nodes/Markdown/Markdown.node.json +++ b/packages/nodes-base/nodes/Markdown/Markdown.node.json @@ -11,6 +11,6 @@ ] }, "subcategories": { - "Core Nodes": ["Helpers", "Data Transformation"] + "Core Nodes": ["Data Transformation"] } } diff --git a/packages/nodes-base/nodes/Merge/Merge.node.json b/packages/nodes-base/nodes/Merge/Merge.node.json index 90e758befdc78..1ea661f4f4220 100644 --- a/packages/nodes-base/nodes/Merge/Merge.node.json +++ b/packages/nodes-base/nodes/Merge/Merge.node.json @@ -43,6 +43,6 @@ }, "alias": ["Join", "Concatenate", "Wait"], "subcategories": { - "Core Nodes": ["Flow"] + "Core Nodes": ["Flow", "Data Transformation"] } } diff --git a/packages/nodes-base/nodes/N8n/N8n.node.json b/packages/nodes-base/nodes/N8n/N8n.node.json index 34fe5023617ed..b46b1cbfba464 100644 --- a/packages/nodes-base/nodes/N8n/N8n.node.json +++ b/packages/nodes-base/nodes/N8n/N8n.node.json @@ -17,6 +17,6 @@ }, "alias": ["Workflow", "Execution"], "subcategories": { - "Core Nodes": ["Helpers", "Flow", "Other Trigger Nodes"] + "Core Nodes": ["Helpers", "Other Trigger Nodes"] } } diff --git a/packages/nodes-base/nodes/RespondToWebhook/RespondToWebhook.node.json b/packages/nodes-base/nodes/RespondToWebhook/RespondToWebhook.node.json index 0df99d6aba89c..19044f48ddc6f 100644 --- a/packages/nodes-base/nodes/RespondToWebhook/RespondToWebhook.node.json +++ b/packages/nodes-base/nodes/RespondToWebhook/RespondToWebhook.node.json @@ -11,6 +11,6 @@ ] }, "subcategories": { - "Core Nodes": ["Flow"] + "Core Nodes": ["Helpers"] } } diff --git a/packages/nodes-base/nodes/RssFeedRead/RssFeedRead.node.json b/packages/nodes-base/nodes/RssFeedRead/RssFeedRead.node.json index 234ca92da921d..f465f25e1dde3 100644 --- a/packages/nodes-base/nodes/RssFeedRead/RssFeedRead.node.json +++ b/packages/nodes-base/nodes/RssFeedRead/RssFeedRead.node.json @@ -21,8 +21,5 @@ "url": "https://n8n.io/blog/why-i-chose-n8n-over-zapier-in-2020/" } ] - }, - "subcategories": { - "Core Nodes": ["Helpers"] } } diff --git a/packages/nodes-base/nodes/Schedule/ScheduleTrigger.node.json b/packages/nodes-base/nodes/Schedule/ScheduleTrigger.node.json index 4f9755e43870a..fd6aa494fb48c 100644 --- a/packages/nodes-base/nodes/Schedule/ScheduleTrigger.node.json +++ b/packages/nodes-base/nodes/Schedule/ScheduleTrigger.node.json @@ -11,8 +11,5 @@ ], "generic": [] }, - "alias": ["Time", "Scheduler", "Polling", "Cron", "Interval"], - "subcategories": { - "Core Nodes": ["Flow"] - } + "alias": ["Time", "Scheduler", "Polling", "Cron", "Interval"] } diff --git a/packages/nodes-base/nodes/SseTrigger/SseTrigger.node.json b/packages/nodes-base/nodes/SseTrigger/SseTrigger.node.json index 3a46546a738cc..7965e4c1bd010 100644 --- a/packages/nodes-base/nodes/SseTrigger/SseTrigger.node.json +++ b/packages/nodes-base/nodes/SseTrigger/SseTrigger.node.json @@ -18,6 +18,6 @@ ] }, "subcategories": { - "Core Nodes": ["Flow", "Other Trigger Nodes"] + "Core Nodes": ["Other Trigger Nodes"] } } diff --git a/packages/nodes-base/nodes/StopAndError/StopAndError.node.json b/packages/nodes-base/nodes/StopAndError/StopAndError.node.json index 29e19dc814bc8..b46fd906bd971 100644 --- a/packages/nodes-base/nodes/StopAndError/StopAndError.node.json +++ b/packages/nodes-base/nodes/StopAndError/StopAndError.node.json @@ -12,6 +12,6 @@ }, "alias": ["Throw error", "Error", "Exception"], "subcategories": { - "Core Nodes": ["Helpers"] + "Core Nodes": ["Flow"] } } diff --git a/packages/nodes-base/nodes/Wait/Wait.node.json b/packages/nodes-base/nodes/Wait/Wait.node.json index 6e8098d3efd88..73debf51a21db 100644 --- a/packages/nodes-base/nodes/Wait/Wait.node.json +++ b/packages/nodes-base/nodes/Wait/Wait.node.json @@ -23,6 +23,6 @@ }, "alias": ["pause", "sleep", "delay", "timeout"], "subcategories": { - "Core Nodes": ["Flow"] + "Core Nodes": ["Helpers", "Flow"] } } diff --git a/packages/nodes-base/nodes/Webhook/Webhook.node.json b/packages/nodes-base/nodes/Webhook/Webhook.node.json index 20502304293c4..cc6961da5c751 100644 --- a/packages/nodes-base/nodes/Webhook/Webhook.node.json +++ b/packages/nodes-base/nodes/Webhook/Webhook.node.json @@ -83,6 +83,6 @@ }, "alias": ["HTTP", "API", "Build", "WH"], "subcategories": { - "Core Nodes": ["Flow"] + "Core Nodes": ["Helpers"] } } diff --git a/packages/workflow/src/NodeHelpers.ts b/packages/workflow/src/NodeHelpers.ts index 32c6e3ce35a20..fd1ddf99007ae 100644 --- a/packages/workflow/src/NodeHelpers.ts +++ b/packages/workflow/src/NodeHelpers.ts @@ -1767,6 +1767,7 @@ export function getVersionedNodeTypeAll(object: IVersionedNodeType | INodeType): Object.values(object.nodeVersions) .map((element) => { element.description.name = object.description.name; + element.description.codex = object.description.codex; return element; }) .reverse(), From c6bc663844df8d256a557f767cb48566deb96631 Mon Sep 17 00:00:00 2001 From: Elias Meire Date: Mon, 4 Dec 2023 16:45:57 +0100 Subject: [PATCH 09/17] Sort items in sections alphabetically --- packages/editor-ui/src/components/Node/NodeCreator/utils.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/editor-ui/src/components/Node/NodeCreator/utils.ts b/packages/editor-ui/src/components/Node/NodeCreator/utils.ts index 02764a71d9bfd..e2661a9bbda93 100644 --- a/packages/editor-ui/src/components/Node/NodeCreator/utils.ts +++ b/packages/editor-ui/src/components/Node/NodeCreator/utils.ts @@ -101,16 +101,16 @@ export function groupItemsInSections( type: 'section', key: section.key, title: section.title, - children: itemsBySection[section.key], + children: sortNodeCreateElements(itemsBySection[section.key] ?? []), }), ) .concat({ type: 'section', key: 'other', title: i18n.baseText('nodeCreator.sectionNames.other'), - children: itemsBySection.other, + children: sortNodeCreateElements(itemsBySection.other ?? []), }) - .filter((section) => section.children); + .filter((section) => section.children.length > 0); if (result.length <= 1) { return items; From 7b1e02e285f5c998511d85e69729c0118ead863d Mon Sep 17 00:00:00 2001 From: Giulio Andreini Date: Tue, 5 Dec 2023 16:21:42 +0100 Subject: [PATCH 10/17] Copy tweaks to nodes descriptions. --- packages/nodes-base/nodes/RenameKeys/RenameKeys.node.ts | 2 +- packages/nodes-base/nodes/Set/v2/SetV2.node.ts | 2 +- .../nodes/Transform/RemoveDuplicates/RemoveDuplicates.node.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/nodes-base/nodes/RenameKeys/RenameKeys.node.ts b/packages/nodes-base/nodes/RenameKeys/RenameKeys.node.ts index cbc44fbef7aa8..d3a8d1ba71916 100644 --- a/packages/nodes-base/nodes/RenameKeys/RenameKeys.node.ts +++ b/packages/nodes-base/nodes/RenameKeys/RenameKeys.node.ts @@ -23,7 +23,7 @@ export class RenameKeys implements INodeType { icon: 'fa:edit', group: ['transform'], version: 1, - description: 'Renames keys', + description: 'Update item field names', defaults: { name: 'Rename Keys', color: '#772244', diff --git a/packages/nodes-base/nodes/Set/v2/SetV2.node.ts b/packages/nodes-base/nodes/Set/v2/SetV2.node.ts index 78135990fa6d3..6bf13b9547297 100644 --- a/packages/nodes-base/nodes/Set/v2/SetV2.node.ts +++ b/packages/nodes-base/nodes/Set/v2/SetV2.node.ts @@ -22,7 +22,7 @@ const versionDescription: INodeTypeDescription = { icon: 'fa:pen', group: ['input'], version: [3, 3.1, 3.2], - description: 'Change fields of items, or create new fields', + description: 'Modify, add, or remove item fields', subtitle: '={{$parameter["mode"]}}', defaults: { name: 'Edit Fields', diff --git a/packages/nodes-base/nodes/Transform/RemoveDuplicates/RemoveDuplicates.node.ts b/packages/nodes-base/nodes/Transform/RemoveDuplicates/RemoveDuplicates.node.ts index 79b7b80f8aad2..2d914fdf34b18 100644 --- a/packages/nodes-base/nodes/Transform/RemoveDuplicates/RemoveDuplicates.node.ts +++ b/packages/nodes-base/nodes/Transform/RemoveDuplicates/RemoveDuplicates.node.ts @@ -20,7 +20,7 @@ export class RemoveDuplicates implements INodeType { group: ['transform'], subtitle: '', version: 1, - description: 'Remove extra items that are similar', + description: 'Delete items with matching field values', defaults: { name: 'Remove Duplicates', }, From 8647fd10a40d9a7e9df05662745995398fadd2c1 Mon Sep 17 00:00:00 2001 From: Elias Meire Date: Tue, 5 Dec 2023 16:57:36 +0100 Subject: [PATCH 11/17] Fix sorting (behavior is different across browsers) --- .../components/Node/NodeCreator/composables/useViewStacks.ts | 2 +- packages/editor-ui/src/components/Node/NodeCreator/utils.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/editor-ui/src/components/Node/NodeCreator/composables/useViewStacks.ts b/packages/editor-ui/src/components/Node/NodeCreator/composables/useViewStacks.ts index b1f58dce935eb..1f2eeb0cee4f9 100644 --- a/packages/editor-ui/src/components/Node/NodeCreator/composables/useViewStacks.ts +++ b/packages/editor-ui/src/components/Node/NodeCreator/composables/useViewStacks.ts @@ -222,7 +222,7 @@ export const useViewStacks = defineStore('nodeCreatorViewStacks', () => { // Sort only if non-root view if (!stack.items) { - sortNodeCreateElements(stackItems); + stackItems = sortNodeCreateElements(stackItems); } updateCurrentViewStack({ baselineItems: stackItems }); diff --git a/packages/editor-ui/src/components/Node/NodeCreator/utils.ts b/packages/editor-ui/src/components/Node/NodeCreator/utils.ts index e2661a9bbda93..237a253c9011a 100644 --- a/packages/editor-ui/src/components/Node/NodeCreator/utils.ts +++ b/packages/editor-ui/src/components/Node/NodeCreator/utils.ts @@ -59,7 +59,7 @@ export function subcategorizeItems(items: SimplifiedNodeType[]) { export function sortNodeCreateElements(nodes: INodeCreateElement[]) { return nodes.sort((a, b) => { - if (a.type !== 'node' || b.type !== 'node') return -1; + if (a.type !== 'node' || b.type !== 'node') return 0; const displayNameA = a.properties?.displayName?.toLowerCase() || a.key; const displayNameB = b.properties?.displayName?.toLowerCase() || b.key; From 46c3f7bd3b9857be08b8e0491b3e98d4b88289cc Mon Sep 17 00:00:00 2001 From: Elias Meire Date: Wed, 6 Dec 2023 13:56:29 +0100 Subject: [PATCH 12/17] Update packages/nodes-base/nodes/Transform/RemoveDuplicates/RemoveDuplicates.node.json Co-authored-by: Deborah --- .../nodes/Transform/RemoveDuplicates/RemoveDuplicates.node.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/nodes-base/nodes/Transform/RemoveDuplicates/RemoveDuplicates.node.json b/packages/nodes-base/nodes/Transform/RemoveDuplicates/RemoveDuplicates.node.json index 7dbdfc40865e4..5130b9f4067a0 100644 --- a/packages/nodes-base/nodes/Transform/RemoveDuplicates/RemoveDuplicates.node.json +++ b/packages/nodes-base/nodes/Transform/RemoveDuplicates/RemoveDuplicates.node.json @@ -7,7 +7,7 @@ "resources": { "primaryDocumentation": [ { - "url": "https://docs.n8n.io/integrations/builtin/core-nodes/n8n-nodes-base.removeDuplicates/" + "url": "https://docs.n8n.io/integrations/builtin/core-nodes/n8n-nodes-base.removeduplicates/" } ], "generic": [] From feb163a1ddd6ee03356a4c212150b81826704bb0 Mon Sep 17 00:00:00 2001 From: Elias Meire Date: Wed, 6 Dec 2023 13:56:34 +0100 Subject: [PATCH 13/17] Update packages/nodes-base/nodes/Transform/SplitOut/SplitOut.node.json Co-authored-by: Deborah --- packages/nodes-base/nodes/Transform/SplitOut/SplitOut.node.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/nodes-base/nodes/Transform/SplitOut/SplitOut.node.json b/packages/nodes-base/nodes/Transform/SplitOut/SplitOut.node.json index 2afc6ec826b0f..8cc55b4864643 100644 --- a/packages/nodes-base/nodes/Transform/SplitOut/SplitOut.node.json +++ b/packages/nodes-base/nodes/Transform/SplitOut/SplitOut.node.json @@ -7,7 +7,7 @@ "resources": { "primaryDocumentation": [ { - "url": "https://docs.n8n.io/integrations/builtin/core-nodes/n8n-nodes-base.splitOut/" + "url": "https://docs.n8n.io/integrations/builtin/core-nodes/n8n-nodes-base.splitout/" } ], "generic": [] From bbf2db0d266f72e7ba7ea70b9d38a37ef0691e35 Mon Sep 17 00:00:00 2001 From: Giulio Andreini Date: Wed, 6 Dec 2023 15:15:52 +0100 Subject: [PATCH 14/17] Icons updates. --- .../nodes/Transform/Aggregate/aggregate.svg | 14 ++++++------- .../nodes/Transform/Limit/limit.svg | 20 +++++++++++++------ .../RemoveDuplicates/removeDuplicates.svg | 15 ++++++++++++-- .../nodes-base/nodes/Transform/Sort/sort.svg | 6 ++++-- .../nodes/Transform/SplitOut/splitOut.svg | 17 +++++++++++----- 5 files changed, 50 insertions(+), 22 deletions(-) diff --git a/packages/nodes-base/nodes/Transform/Aggregate/aggregate.svg b/packages/nodes-base/nodes/Transform/Aggregate/aggregate.svg index a55ebf954cf64..35ee5d4611f32 100644 --- a/packages/nodes-base/nodes/Transform/Aggregate/aggregate.svg +++ b/packages/nodes-base/nodes/Transform/Aggregate/aggregate.svg @@ -1,13 +1,13 @@ - - - - - - + + + + + + - + diff --git a/packages/nodes-base/nodes/Transform/Limit/limit.svg b/packages/nodes-base/nodes/Transform/Limit/limit.svg index ef9b84010f534..6e17fc2f1b4c1 100644 --- a/packages/nodes-base/nodes/Transform/Limit/limit.svg +++ b/packages/nodes-base/nodes/Transform/Limit/limit.svg @@ -1,8 +1,16 @@ - - - - - - + + + + + + + + + + + + + + diff --git a/packages/nodes-base/nodes/Transform/RemoveDuplicates/removeDuplicates.svg b/packages/nodes-base/nodes/Transform/RemoveDuplicates/removeDuplicates.svg index ce3b3f44b197d..2d5162b5b63c5 100644 --- a/packages/nodes-base/nodes/Transform/RemoveDuplicates/removeDuplicates.svg +++ b/packages/nodes-base/nodes/Transform/RemoveDuplicates/removeDuplicates.svg @@ -1,4 +1,15 @@ - - + + + + + + + + + + + + + diff --git a/packages/nodes-base/nodes/Transform/Sort/sort.svg b/packages/nodes-base/nodes/Transform/Sort/sort.svg index 3b1154a00cdd2..0f53ab94d1623 100644 --- a/packages/nodes-base/nodes/Transform/Sort/sort.svg +++ b/packages/nodes-base/nodes/Transform/Sort/sort.svg @@ -1,4 +1,6 @@ - - + + + + diff --git a/packages/nodes-base/nodes/Transform/SplitOut/splitOut.svg b/packages/nodes-base/nodes/Transform/SplitOut/splitOut.svg index 34437cd3360d2..d4aeb22739462 100644 --- a/packages/nodes-base/nodes/Transform/SplitOut/splitOut.svg +++ b/packages/nodes-base/nodes/Transform/SplitOut/splitOut.svg @@ -1,7 +1,14 @@ - - - - - + + + + + + + + + + + + From c5281ca94ceb106866302b26bd744d20c842905b Mon Sep 17 00:00:00 2001 From: Elias Meire Date: Wed, 6 Dec 2023 17:23:59 +0100 Subject: [PATCH 15/17] Add test for node creator sorting --- .../Node/NodeCreator/__tests__/utils.test.ts | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/packages/editor-ui/src/components/Node/NodeCreator/__tests__/utils.test.ts b/packages/editor-ui/src/components/Node/NodeCreator/__tests__/utils.test.ts index a89a5e2202c9d..6f7b45cc5cbf0 100644 --- a/packages/editor-ui/src/components/Node/NodeCreator/__tests__/utils.test.ts +++ b/packages/editor-ui/src/components/Node/NodeCreator/__tests__/utils.test.ts @@ -1,6 +1,6 @@ import type { SectionCreateElement } from '@/Interface'; -import { groupItemsInSections } from '../utils'; -import { mockNodeCreateElement } from './utils'; +import { groupItemsInSections, sortNodeCreateElements } from '../utils'; +import { mockActionCreateElement, mockNodeCreateElement, mockSectionCreateElement } from './utils'; describe('NodeCreator - utils', () => { describe('groupItemsInSections', () => { @@ -46,4 +46,20 @@ describe('NodeCreator - utils', () => { expect(result).toEqual([node1, node2, node3]); }); }); + + describe('sortNodeCreateElements', () => { + it('should sort nodes alphabetically by displayName', () => { + const node1 = mockNodeCreateElement({ key: 'newNode' }, { displayName: 'xyz' }); + const node2 = mockNodeCreateElement({ key: 'popularNode' }, { displayName: 'abc' }); + const node3 = mockNodeCreateElement({ key: 'otherNode' }, { displayName: 'ABC' }); + expect(sortNodeCreateElements([node1, node2, node3])).toEqual([node2, node3, node1]); + }); + + it('should not change order for other types (sections, actions)', () => { + const node1 = mockSectionCreateElement(); + const node2 = mockActionCreateElement(); + const node3 = mockSectionCreateElement(); + expect(sortNodeCreateElements([node1, node2, node3])).toEqual([node1, node2, node3]); + }); + }); }); From c9c2e1b4b7e1855c2a9b529b48acc90e013f92cc Mon Sep 17 00:00:00 2001 From: Elias Meire Date: Fri, 8 Dec 2023 09:48:58 +0100 Subject: [PATCH 16/17] Migrate Item Lists nodes in e2e tests --- cypress/e2e/14-mapping.cy.ts | 7 ++----- cypress/e2e/24-ndv-paired-item.cy.ts | 4 ++-- cypress/e2e/4-node-creator.cy.ts | 4 ++-- cypress/fixtures/Test_workflow_5.json | 13 ++++++------- 4 files changed, 12 insertions(+), 16 deletions(-) diff --git a/cypress/e2e/14-mapping.cy.ts b/cypress/e2e/14-mapping.cy.ts index af39f6898ba18..c547383e3e7ee 100644 --- a/cypress/e2e/14-mapping.cy.ts +++ b/cypress/e2e/14-mapping.cy.ts @@ -235,11 +235,8 @@ describe('Data mapping', () => { ndv.actions.close(); - workflowPage.actions.addNodeToCanvas('Item Lists'); - workflowPage.actions.openNode('Item Lists'); - - ndv.getters.parameterInput('operation').click(); - getVisibleSelect().find('li').contains('Sort').click(); + workflowPage.actions.addNodeToCanvas('Sort'); + workflowPage.actions.openNode('Sort'); ndv.getters.nodeParameters().find('button').contains('Add Field To Sort By').click(); diff --git a/cypress/e2e/24-ndv-paired-item.cy.ts b/cypress/e2e/24-ndv-paired-item.cy.ts index d1509db9e63c5..58fa0fdb63228 100644 --- a/cypress/e2e/24-ndv-paired-item.cy.ts +++ b/cypress/e2e/24-ndv-paired-item.cy.ts @@ -19,7 +19,7 @@ describe('NDV', () => { workflowPage.actions.executeWorkflow(); - workflowPage.actions.openNode('Item Lists'); + workflowPage.actions.openNode('Sort'); ndv.getters.inputPanel().contains('6 items').should('exist'); ndv.getters.outputPanel().contains('6 items').should('exist'); @@ -92,7 +92,7 @@ describe('NDV', () => { ndv.getters.outputHoveringItem().should('have.text', '1000'); ndv.getters.parameterExpressionPreview('value').should('include.text', '1000'); - ndv.actions.selectInputNode('Item Lists'); + ndv.actions.selectInputNode('Sort'); ndv.actions.changeOutputRunSelector('1 of 2 (6 items)'); ndv.getters.backToCanvas().realHover(); // reset to default hover diff --git a/cypress/e2e/4-node-creator.cy.ts b/cypress/e2e/4-node-creator.cy.ts index 737fc52328be5..828fb27cd388e 100644 --- a/cypress/e2e/4-node-creator.cy.ts +++ b/cypress/e2e/4-node-creator.cy.ts @@ -316,7 +316,7 @@ describe('Node Creator', () => { NDVModal.actions.close(); WorkflowPage.getters.canvasNodes().should('have.length', 2); WorkflowPage.actions.zoomToFit(); - WorkflowPage.actions.addNodeBetweenNodes('n8n', 'n8n1', 'Item Lists', 'Summarize'); + WorkflowPage.actions.addNodeBetweenNodes('n8n', 'n8n1', 'Summarize'); WorkflowPage.getters.canvasNodes().should('have.length', 3); }); }); @@ -410,7 +410,7 @@ describe('Node Creator', () => { nodeCreatorFeature.getters.searchBar().find('input').clear().type('js'); nodeCreatorFeature.getters.nodeItemName().first().should('have.text', 'Code'); - nodeCreatorFeature.getters.nodeItemName().eq(1).should('have.text', 'Item Lists'); + nodeCreatorFeature.getters.nodeItemName().eq(1).should('have.text', 'Edit Fields (Set)'); nodeCreatorFeature.getters.searchBar().find('input').clear().type('fi'); nodeCreatorFeature.getters.nodeItemName().first().should('have.text', 'Filter'); diff --git a/cypress/fixtures/Test_workflow_5.json b/cypress/fixtures/Test_workflow_5.json index 6b87fc33b70a1..e2588da3dfcee 100644 --- a/cypress/fixtures/Test_workflow_5.json +++ b/cypress/fixtures/Test_workflow_5.json @@ -50,7 +50,6 @@ }, { "parameters": { - "operation": "sort", "sortFieldsUi": { "sortField": [ { @@ -61,9 +60,9 @@ "options": {} }, "id": "555a150c-d735-4331-b628-c1f1cfed2da1", - "name": "Item Lists", - "type": "n8n-nodes-base.itemLists", - "typeVersion": 2, + "name": "Sort", + "type": "n8n-nodes-base.sort", + "typeVersion": 1, "position": [ -280, 580 @@ -182,7 +181,7 @@ "main": [ [ { - "node": "Item Lists", + "node": "Sort", "type": "main", "index": 0 } @@ -216,7 +215,7 @@ ] ] }, - "Item Lists": { + "Sort": { "main": [ [ { @@ -289,4 +288,4 @@ ] } } -} \ No newline at end of file +} From 59f9eeda48763d574e58cb0fc7963675efd64a1d Mon Sep 17 00:00:00 2001 From: Elias Meire Date: Fri, 8 Dec 2023 10:49:31 +0100 Subject: [PATCH 17/17] Fix e2e tests --- cypress/e2e/11-inline-expression-editor.cy.ts | 2 +- cypress/e2e/26-resource-locator.cy.ts | 8 ++++---- cypress/e2e/9-expression-editor-modal.cy.ts | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cypress/e2e/11-inline-expression-editor.cy.ts b/cypress/e2e/11-inline-expression-editor.cy.ts index f7e192e06c8bd..7f1b97b03c55f 100644 --- a/cypress/e2e/11-inline-expression-editor.cy.ts +++ b/cypress/e2e/11-inline-expression-editor.cy.ts @@ -67,6 +67,6 @@ describe('Inline expression editor', () => { WorkflowPage.getters.inlineExpressionEditorInput().type('{{'); // Resolving $parameter is slow, especially on CI runner WorkflowPage.getters.inlineExpressionEditorInput().type('$parameter["operation"]'); - WorkflowPage.getters.inlineExpressionEditorOutput().contains(/^get$/); + WorkflowPage.getters.inlineExpressionEditorOutput().should('have.text', 'getAll'); }); }); diff --git a/cypress/e2e/26-resource-locator.cy.ts b/cypress/e2e/26-resource-locator.cy.ts index c556ba983e7de..60d33374d779f 100644 --- a/cypress/e2e/26-resource-locator.cy.ts +++ b/cypress/e2e/26-resource-locator.cy.ts @@ -16,7 +16,7 @@ describe('Resource Locator', () => { it('should render both RLC components in google sheets', () => { workflowPage.actions.addInitialNodeToCanvas('Manual'); - workflowPage.actions.addNodeToCanvas('Google Sheets', true, true); + workflowPage.actions.addNodeToCanvas('Google Sheets', true, true, 'Update row in sheet'); ndv.getters.resourceLocator('documentId').should('be.visible'); ndv.getters.resourceLocator('sheetName').should('be.visible'); ndv.getters @@ -31,7 +31,7 @@ describe('Resource Locator', () => { it('should show appropriate error when credentials are not set', () => { workflowPage.actions.addInitialNodeToCanvas('Manual'); - workflowPage.actions.addNodeToCanvas('Google Sheets', true, true); + workflowPage.actions.addNodeToCanvas('Google Sheets', true, true, 'Update row in sheet'); ndv.getters.resourceLocator('documentId').should('be.visible'); ndv.getters.resourceLocatorInput('documentId').click(); ndv.getters.resourceLocatorErrorMessage().should('contain', NO_CREDENTIALS_MESSAGE); @@ -39,7 +39,7 @@ describe('Resource Locator', () => { it('should show appropriate error when credentials are not valid', () => { workflowPage.actions.addInitialNodeToCanvas('Manual'); - workflowPage.actions.addNodeToCanvas('Google Sheets', true, true); + workflowPage.actions.addNodeToCanvas('Google Sheets', true, true, 'Update row in sheet'); workflowPage.getters.nodeCredentialsSelect().click(); // Add oAuth credentials getVisibleSelect().find('li').last().click(); @@ -54,7 +54,7 @@ describe('Resource Locator', () => { it('should reset resource locator when dependent field is changed', () => { workflowPage.actions.addInitialNodeToCanvas('Manual'); - workflowPage.actions.addNodeToCanvas('Google Sheets', true, true); + workflowPage.actions.addNodeToCanvas('Google Sheets', true, true, 'Update row in sheet'); ndv.actions.setRLCValue('documentId', '123'); ndv.actions.setRLCValue('sheetName', '123'); ndv.actions.setRLCValue('documentId', '321'); diff --git a/cypress/e2e/9-expression-editor-modal.cy.ts b/cypress/e2e/9-expression-editor-modal.cy.ts index 46affa0d62a5e..3cd00d1f6886e 100644 --- a/cypress/e2e/9-expression-editor-modal.cy.ts +++ b/cypress/e2e/9-expression-editor-modal.cy.ts @@ -57,6 +57,6 @@ describe('Expression editor modal', () => { it('should resolve $parameter[]', () => { WorkflowPage.getters.expressionModalInput().clear(); WorkflowPage.getters.expressionModalInput().type('{{ $parameter["operation"]'); - WorkflowPage.getters.expressionModalOutput().contains(/^get$/); + WorkflowPage.getters.expressionModalOutput().should('have.text', 'getAll'); }); });