Skip to content

Commit

Permalink
feat(Item Lists Node): Split merge binary data (#7297)
Browse files Browse the repository at this point in the history 
Github issue / Community forum post (link here to close automatically):

---------

Co-authored-by: Marcus <[email protected]>
  • Loading branch information
@michael-radency @maspio
michael-radency and maspio authored Oct 11, 2023
1 parent e2c3c7a commit 965db8f
Show file tree
Hide file tree
Showing 3 changed files with 195 additions and 61 deletions.
69 changes: 58 additions & 11 deletions packages/nodes-base/nodes/ItemLists/V3/actions/itemList/concatenateItems.operation.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ import get from 'lodash/get';
import isEmpty from 'lodash/isEmpty';
import set from 'lodash/set';

import { prepareFieldsArray } from '../../helpers/utils';
import { addBinariesToItem, prepareFieldsArray } from '../../helpers/utils';
import { disableDotNotationBoolean } from '../common.descriptions';

const properties: INodeProperties[] = [
Expand Down Expand Up @@ -159,20 +159,47 @@ const properties: INodeProperties[] = [
type: 'collection',
placeholder: 'Add Field',
default: {},
displayOptions: {
hide: {
aggregate: ['aggregateAllItemData'],
},
},
options: [
disableDotNotationBoolean,
{
...disableDotNotationBoolean,
displayOptions: {
hide: {
'/aggregate': ['aggregateAllItemData'],
},
},
},
{
displayName: 'Merge Lists',
name: 'mergeLists',
type: 'boolean',
default: false,
description:
'Whether to merge the output into a single flat list (rather than a list of lists), if the field to aggregate is a list',
displayOptions: {
hide: {
'/aggregate': ['aggregateAllItemData'],
},
},
},
{
displayName: 'Include Binaries',
name: 'includeBinaries',
type: 'boolean',
default: false,
description: 'Whether to include the binary data in the new item',
},
{
displayName: 'Keep Only Unique Binaries',
name: 'keepOnlyUnique',
type: 'boolean',
default: false,
description:
'Whether to keep only unique binaries by comparing mime types, file types, file sizes and file extensions',
displayOptions: {
show: {
includeBinaries: [true],
},
},
},
{
displayName: 'Keep Missing And Null Values',
Expand All @@ -181,6 +208,11 @@ const properties: INodeProperties[] = [
default: false,
description:
'Whether to add a null entry to the aggregated list when there is a missing or null value',
displayOptions: {
hide: {
'/aggregate': ['aggregateAllItemData'],
},
},
},
],
},
Expand All @@ -199,7 +231,7 @@ export async function execute(
this: IExecuteFunctions,
items: INodeExecutionData[],
): Promise<INodeExecutionData[]> {
const returnData: INodeExecutionData[] = [];
let returnData: INodeExecutionData = { json: {}, pairedItem: [] };

const aggregate = this.getNodeParameter('aggregate', 0, '') as string;

Expand Down Expand Up @@ -305,7 +337,7 @@ export async function execute(
}
}

returnData.push(newItem);
returnData = newItem;
} else {
let newItems: IDataObject[] = items.map((item) => item.json);
let pairedItem: IPairedItemData[] = [];
Expand Down Expand Up @@ -353,8 +385,23 @@ export async function execute(
}

const output: INodeExecutionData = { json: { [destinationFieldName]: newItems }, pairedItem };
returnData.push(output);

returnData = output;
}

const includeBinaries = this.getNodeParameter('options.includeBinaries', 0, false) as boolean;

if (includeBinaries) {
const pairedItems = (returnData.pairedItem || []) as IPairedItemData[];

const aggregatedItems = pairedItems.map((item) => {
return items[item.item];
});

const keepOnlyUnique = this.getNodeParameter('options.keepOnlyUnique', 0, false) as boolean;

addBinariesToItem(returnData, aggregatedItems, keepOnlyUnique);
}

return returnData;
return [returnData];
}
115 changes: 71 additions & 44 deletions packages/nodes-base/nodes/ItemLists/V3/actions/itemList/splitOutItems.operation.ts
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import type {
IBinaryData,
IDataObject,
IExecuteFunctions,
INodeExecutionData,
Expand All @@ -20,7 +21,9 @@ const properties: INodeProperties[] = [
type: 'string',
default: '',
required: true,
description: 'The name of the input fields to break out into separate items',
placeholder: 'Drag fields from the left or type their names',
description:
'The name of the input fields to break out into separate items. Separate multiple field names by commas. For binary data, use $binary.',
requiresDataPath: 'multiple',
},
{
Expand Down Expand Up @@ -74,6 +77,13 @@ const properties: INodeProperties[] = [
default: '',
description: 'The field in the output under which to put the split field contents',
},
{
displayName: 'Include Binary',
name: 'includeBinary',
type: 'boolean',
default: false,
description: 'Whether to include the binary data in the new items',
},
],
},
];
Expand All @@ -96,16 +106,13 @@ export async function execute(
for (let i = 0; i < items.length; i++) {
const fieldsToSplitOut = (this.getNodeParameter('fieldToSplitOut', i) as string)
.split(',')
.map((field) => field.trim());
const disableDotNotation = this.getNodeParameter(
'options.disableDotNotation',
0,
false,
) as boolean;

const destinationFields = (
this.getNodeParameter('options.destinationFieldName', i, '') as string
)
.map((field) => field.trim().replace(/^\$json\./, ''));

const options = this.getNodeParameter('options', i, {});

const disableDotNotation = options.disableDotNotation as boolean;

const destinationFields = ((options.destinationFieldName as string) || '')
.split(',')
.filter((field) => field.trim() !== '')
.map((field) => field.trim());
Expand All @@ -125,54 +132,71 @@ export async function execute(
const multiSplit = fieldsToSplitOut.length > 1;

const item = { ...items[i].json };
const splited: IDataObject[] = [];
const splited: INodeExecutionData[] = [];
for (const [entryIndex, fieldToSplitOut] of fieldsToSplitOut.entries()) {
const destinationFieldName = destinationFields[entryIndex] || '';

let arrayToSplit;
if (!disableDotNotation) {
arrayToSplit = get(item, fieldToSplitOut);
let entityToSplit: IDataObject[] = [];

if (fieldToSplitOut === '$binary') {
entityToSplit = Object.entries(items[i].binary || {}).map(([key, value]) => ({
[key]: value,
}));
} else {
arrayToSplit = item[fieldToSplitOut];
}
if (!disableDotNotation) {
entityToSplit = get(item, fieldToSplitOut) as IDataObject[];
} else {
entityToSplit = item[fieldToSplitOut] as IDataObject[];
}

if (arrayToSplit === undefined) {
arrayToSplit = [];
}
if (entityToSplit === undefined) {
entityToSplit = [];
}

if (typeof arrayToSplit !== 'object' || arrayToSplit === null) {
arrayToSplit = [arrayToSplit];
}
if (typeof entityToSplit !== 'object' || entityToSplit === null) {
entityToSplit = [entityToSplit];
}

if (!Array.isArray(arrayToSplit)) {
arrayToSplit = Object.values(arrayToSplit);
if (!Array.isArray(entityToSplit)) {
entityToSplit = Object.values(entityToSplit);
}
}

for (const [elementIndex, element] of arrayToSplit.entries()) {
for (const [elementIndex, element] of entityToSplit.entries()) {
if (splited[elementIndex] === undefined) {
splited[elementIndex] = {};
splited[elementIndex] = { json: {}, pairedItem: { item: i } };
}

const fieldName = destinationFieldName || fieldToSplitOut;

if (fieldToSplitOut === '$binary') {
if (splited[elementIndex].binary === undefined) {
splited[elementIndex].binary = {};
}
splited[elementIndex].binary![Object.keys(element)[0]] = Object.values(
element,
)[0] as IBinaryData;

continue;
}

if (typeof element === 'object' && element !== null && include === 'noOtherFields') {
if (destinationFieldName === '' && !multiSplit) {
splited[elementIndex] = { ...splited[elementIndex], ...element };
splited[elementIndex] = {
json: { ...splited[elementIndex].json, ...element },
pairedItem: { item: i },
};
} else {
splited[elementIndex][fieldName] = element;
splited[elementIndex].json[fieldName] = element;
}
} else {
splited[elementIndex][fieldName] = element;
splited[elementIndex].json[fieldName] = element;
}
}
}

for (const splitEntry of splited) {
let newItem: IDataObject = {};

if (include === 'noOtherFields') {
newItem = splitEntry;
}
let newItem: INodeExecutionData = splitEntry;

if (include === 'allOtherFields') {
const itemCopy = deepCopy(item);
Expand All @@ -183,7 +207,7 @@ export async function execute(
delete itemCopy[fieldToSplitOut];
}
}
newItem = { ...itemCopy, ...splitEntry };
newItem.json = { ...itemCopy, ...splitEntry.json };
}

if (include === 'selectedOtherFields') {
Expand All @@ -200,21 +224,24 @@ export async function execute(

for (const field of fieldsToInclude) {
if (!disableDotNotation) {
splitEntry[field] = get(item, field);
splitEntry.json[field] = get(item, field);
} else {
splitEntry[field] = item[field];
splitEntry.json[field] = item[field];
}
}

newItem = splitEntry;
}

returnData.push({
json: newItem,
pairedItem: {
item: i,
},
});
const includeBinary = options.includeBinary as boolean;

if (includeBinary) {
if (items[i].binary && !newItem.binary) {
newItem.binary = items[i].binary;
}
}

returnData.push(newItem);
}
}

Expand Down
72 changes: 66 additions & 6 deletions packages/nodes-base/nodes/ItemLists/V3/helpers/utils.ts
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,12 @@
import { NodeVM } from '@n8n/vm2';
import {
NodeOperationError,
type IDataObject,
type IExecuteFunctions,
type INode,
type INodeExecutionData,
import type {
IDataObject,
IExecuteFunctions,
IBinaryData,
INode,
INodeExecutionData,
} from 'n8n-workflow';
import { NodeOperationError } from 'n8n-workflow';

import get from 'lodash/get';
import isEqual from 'lodash/isEqual';
Expand Down Expand Up @@ -87,3 +88,62 @@ export function sortByCode(

return vm.run(`module.exports = items.sort((a, b) => { ${code} })`);
}

type PartialBinaryData = Omit<IBinaryData, 'data'>;
const isBinaryUniqueSetup = () => {
const binaries: PartialBinaryData[] = [];
return (binary: IBinaryData) => {
for (const existingBinary of binaries) {
if (
existingBinary.mimeType === binary.mimeType &&
existingBinary.fileType === binary.fileType &&
existingBinary.fileSize === binary.fileSize &&
existingBinary.fileExtension === binary.fileExtension
) {
return false;
}
}

binaries.push({
mimeType: binary.mimeType,
fileType: binary.fileType,
fileSize: binary.fileSize,
fileExtension: binary.fileExtension,
});

return true;
};
};

export function addBinariesToItem(
newItem: INodeExecutionData,
items: INodeExecutionData[],
uniqueOnly?: boolean,
) {
const isBinaryUnique = uniqueOnly ? isBinaryUniqueSetup() : undefined;

for (const item of items) {
if (item.binary === undefined) continue;

for (const key of Object.keys(item.binary)) {
if (!newItem.binary) newItem.binary = {};
let binaryKey = key;
const binary = item.binary[key];

if (isBinaryUnique && !isBinaryUnique(binary)) {
continue;
}

// If the binary key already exists add a suffix to it
let i = 1;
while (newItem.binary[binaryKey] !== undefined) {
binaryKey = `${key}_${i}`;
i++;
}

newItem.binary[binaryKey] = binary;
}
}

return newItem;
}

0 comments on commit 965db8f

Please sign in to comment.