diff --git a/packages/@n8n/nodes-langchain/nodes/tools/ToolHttpRequest/ToolHttpRequest.node.ts b/packages/@n8n/nodes-langchain/nodes/tools/ToolHttpRequest/ToolHttpRequest.node.ts new file mode 100644 index 00000000000000..b22dca4c27ba88 --- /dev/null +++ b/packages/@n8n/nodes-langchain/nodes/tools/ToolHttpRequest/ToolHttpRequest.node.ts @@ -0,0 +1,405 @@ +/* eslint-disable n8n-nodes-base/node-dirname-against-convention */ +import type { + IExecuteFunctions, + INodeType, + INodeTypeDescription, + SupplyData, + IHttpRequestMethods, + IHttpRequestOptions, +} from 'n8n-workflow'; +import { NodeConnectionType, NodeOperationError, tryToParseAlphanumericString } from 'n8n-workflow'; + +import { getConnectionHintNoticeField } from '../../../utils/sharedFields'; + +import { + configureHttpRequestFunction, + configureResponseOptimizer, + extractParametersFromText, + prepareToolDescription, + configureToolFunction, + updateParametersAndOptions, +} from './utils'; + +import { + authenticationProperties, + jsonInput, + optimizeResponseProperties, + parametersCollection, + placeholderDefinitionsCollection, + specifyBySelector, +} from './descriptions'; + +import type { PlaceholderDefinition, ToolParameter } from './interfaces'; + +import { DynamicTool } from '@langchain/core/tools'; + +export class ToolHttpRequest implements INodeType { + description: INodeTypeDescription = { + displayName: 'HTTP Request Tool', + name: 'toolHttpRequest', + icon: { light: 'file:httprequest.svg', dark: 'file:httprequest.dark.svg' }, + group: ['output'], + version: 1, + description: 'Makes an HTTP request and returns the response data', + subtitle: '={{ $parameter.toolDescription }}', + defaults: { + name: 'HTTP Request', + }, + credentials: [], + codex: { + categories: ['AI'], + subcategories: { + AI: ['Tools'], + }, + resources: { + primaryDocumentation: [ + { + url: 'https://docs.n8n.io/integrations/builtin/cluster-nodes/sub-nodes/n8n-nodes-langchain.toolhttprequest/', + }, + ], + }, + }, + // eslint-disable-next-line n8n-nodes-base/node-class-description-inputs-wrong-regular-node + inputs: [], + // eslint-disable-next-line n8n-nodes-base/node-class-description-outputs-wrong + outputs: [NodeConnectionType.AiTool], + outputNames: ['Tool'], + properties: [ + getConnectionHintNoticeField([NodeConnectionType.AiAgent]), + { + displayName: 'Description', + name: 'toolDescription', + type: 'string', + description: + 'Explain to LLM what this tool does, better description would allow LLM to produce expected result', + placeholder: 'e.g. Get the current weather in the requested city', + default: '', + typeOptions: { + rows: 3, + }, + }, + { + displayName: 'Method', + name: 'method', + type: 'options', + options: [ + { + name: 'DELETE', + value: 'DELETE', + }, + { + name: 'GET', + value: 'GET', + }, + { + name: 'PATCH', + value: 'PATCH', + }, + { + name: 'POST', + value: 'POST', + }, + { + name: 'PUT', + value: 'PUT', + }, + ], + default: 'GET', + }, + { + displayName: + 'Tip: You can use a {placeholder} for any part of the request to be filled by the model. Provide more context about them in the placeholders section', + name: 'placeholderNotice', + type: 'notice', + default: '', + }, + { + displayName: 'URL', + name: 'url', + type: 'string', + default: '', + required: true, + placeholder: 'e.g. http://www.example.com/{path}', + }, + ...authenticationProperties, + //---------------------------------------------------------------- + { + displayName: 'Send Query Parameters', + name: 'sendQuery', + type: 'boolean', + default: false, + noDataExpression: true, + description: 'Whether the request has query params or not', + }, + { + ...specifyBySelector, + displayName: 'Specify Query Parameters', + name: 'specifyQuery', + displayOptions: { + show: { + sendQuery: [true], + }, + }, + }, + { + ...parametersCollection, + displayName: 'Query Parameters', + name: 'parametersQuery', + displayOptions: { + show: { + sendQuery: [true], + specifyQuery: ['keypair'], + }, + }, + }, + { + ...jsonInput, + name: 'jsonQuery', + displayOptions: { + show: { + sendQuery: [true], + specifyQuery: ['json'], + }, + }, + }, + //---------------------------------------------------------------- + { + displayName: 'Send Headers', + name: 'sendHeaders', + type: 'boolean', + default: false, + noDataExpression: true, + description: 'Whether the request has headers or not', + }, + { + ...specifyBySelector, + displayName: 'Specify Headers', + name: 'specifyHeaders', + displayOptions: { + show: { + sendHeaders: [true], + }, + }, + }, + { + ...parametersCollection, + displayName: 'Header Parameters', + name: 'parametersHeaders', + displayOptions: { + show: { + sendHeaders: [true], + specifyHeaders: ['keypair'], + }, + }, + }, + { + ...jsonInput, + name: 'jsonHeaders', + displayOptions: { + show: { + sendHeaders: [true], + specifyHeaders: ['json'], + }, + }, + }, + //---------------------------------------------------------------- + { + displayName: 'Send Body', + name: 'sendBody', + type: 'boolean', + default: false, + noDataExpression: true, + description: 'Whether the request has body or not', + }, + { + ...specifyBySelector, + displayName: 'Specify Body', + name: 'specifyBody', + displayOptions: { + show: { + sendBody: [true], + }, + }, + }, + { + ...parametersCollection, + displayName: 'Body Parameters', + name: 'parametersBody', + displayOptions: { + show: { + sendBody: [true], + specifyBody: ['keypair'], + }, + }, + }, + { + ...jsonInput, + name: 'jsonBody', + displayOptions: { + show: { + sendBody: [true], + specifyBody: ['json'], + }, + }, + }, + //---------------------------------------------------------------- + placeholderDefinitionsCollection, + ...optimizeResponseProperties, + ], + }; + + async supplyData(this: IExecuteFunctions, itemIndex: number): Promise { + const name = this.getNode().name.replace(/ /g, '_'); + try { + tryToParseAlphanumericString(name); + } catch (error) { + throw new NodeOperationError( + this.getNode(), + 'The name of this tool is not a valid alphanumeric string', + { + itemIndex, + description: + "Only alphanumeric characters and underscores are allowed in the tool's name, and the name cannot start with a number", + }, + ); + } + + const toolDescription = this.getNodeParameter('toolDescription', itemIndex) as string; + const sendQuery = this.getNodeParameter('sendQuery', itemIndex, false) as boolean; + const sendHeaders = this.getNodeParameter('sendHeaders', itemIndex, false) as boolean; + const sendBody = this.getNodeParameter('sendBody', itemIndex, false) as boolean; + + const requestOptions: IHttpRequestOptions = { + method: this.getNodeParameter('method', itemIndex, 'GET') as IHttpRequestMethods, + url: this.getNodeParameter('url', itemIndex) as string, + qs: {}, + headers: {}, + body: {}, + }; + + const authentication = this.getNodeParameter('authentication', itemIndex, 'none') as + | 'predefinedCredentialType' + | 'genericCredentialType' + | 'none'; + + if (authentication !== 'none') { + const domain = new URL(requestOptions.url).hostname; + if (domain.includes('{') && domain.includes('}')) { + throw new NodeOperationError( + this.getNode(), + "Can't use a placeholder for the domain when using authentication", + { + itemIndex, + description: + 'This is for security reasons, to prevent the model accidentally sending your credentials to an unauthorized domain', + }, + ); + } + } + + const httpRequest = await configureHttpRequestFunction(this, authentication, itemIndex); + const optimizeResponse = configureResponseOptimizer(this, itemIndex); + + const rawRequestOptions: { [key: string]: string } = { + qs: '', + headers: '', + body: '', + }; + + const placeholdersDefinitions = ( + this.getNodeParameter( + 'placeholderDefinitions.values', + itemIndex, + [], + ) as PlaceholderDefinition[] + ).map((p) => { + if (p.name.startsWith('{') && p.name.endsWith('}')) { + p.name = p.name.slice(1, -1); + } + return p; + }); + + const toolParameters: ToolParameter[] = []; + + toolParameters.push( + ...extractParametersFromText(placeholdersDefinitions, requestOptions.url, 'path'), + ); + + if (sendQuery) { + updateParametersAndOptions({ + ctx: this, + itemIndex, + toolParameters, + placeholdersDefinitions, + requestOptions, + rawRequestOptions, + requestOptionsProperty: 'qs', + inputTypePropertyName: 'specifyQuery', + jsonPropertyName: 'jsonQuery', + parametersPropertyName: 'parametersQuery.values', + }); + } + + if (sendHeaders) { + updateParametersAndOptions({ + ctx: this, + itemIndex, + toolParameters, + placeholdersDefinitions, + requestOptions, + rawRequestOptions, + requestOptionsProperty: 'headers', + inputTypePropertyName: 'specifyHeaders', + jsonPropertyName: 'jsonHeaders', + parametersPropertyName: 'parametersHeaders.values', + }); + } + + if (sendBody) { + updateParametersAndOptions({ + ctx: this, + itemIndex, + toolParameters, + placeholdersDefinitions, + requestOptions, + rawRequestOptions, + requestOptionsProperty: 'body', + inputTypePropertyName: 'specifyBody', + jsonPropertyName: 'jsonBody', + parametersPropertyName: 'parametersBody.values', + }); + } + + for (const placeholder of placeholdersDefinitions) { + if (!toolParameters.find((parameter) => parameter.name === placeholder.name)) { + throw new NodeOperationError( + this.getNode(), + `Misconfigured placeholder '${placeholder.name}'`, + { + itemIndex, + description: + "This placeholder is defined in the 'Placeholder Definitions' but isn't used anywhere. Either remove the definition, or add the placeholder to a part of the request.", + }, + ); + } + } + + const func = configureToolFunction( + this, + itemIndex, + toolParameters, + requestOptions, + rawRequestOptions, + httpRequest, + optimizeResponse, + ); + + const description = prepareToolDescription(toolDescription, toolParameters); + + const tool = new DynamicTool({ name, description, func }); + + return { + response: tool, + }; + } +} diff --git a/packages/@n8n/nodes-langchain/nodes/tools/ToolHttpRequest/descriptions.ts b/packages/@n8n/nodes-langchain/nodes/tools/ToolHttpRequest/descriptions.ts new file mode 100644 index 00000000000000..43fcf673a17944 --- /dev/null +++ b/packages/@n8n/nodes-langchain/nodes/tools/ToolHttpRequest/descriptions.ts @@ -0,0 +1,401 @@ +import type { INodeProperties } from 'n8n-workflow'; + +export const specifyBySelector: INodeProperties = { + displayName: 'Specify By', + name: 'specifyBy', + type: 'options', + options: [ + { + name: 'Using Fields Below', + value: 'keypair', + }, + { + name: 'Using JSON Below', + value: 'json', + }, + { + name: 'Let Model Specify Entire Body', + value: 'model', + }, + ], + default: 'keypair', +}; + +export const parametersCollection: INodeProperties = { + displayName: 'Parameters', + name: 'parameters', + type: 'fixedCollection', + typeOptions: { + multipleValues: true, + }, + placeholder: 'Add Parameter', + default: { + values: [ + { + name: '', + }, + ], + }, + options: [ + { + name: 'values', + displayName: 'Values', + values: [ + { + displayName: 'Name', + name: 'name', + type: 'string', + default: '', + }, + { + displayName: 'Value Provided', + name: 'valueProvider', + type: 'options', + options: [ + { + // eslint-disable-next-line n8n-nodes-base/node-param-display-name-miscased + name: 'By Model (and is required)', + value: 'modelRequired', + }, + { + // eslint-disable-next-line n8n-nodes-base/node-param-display-name-miscased + name: 'By Model (but is optional)', + value: 'modelOptional', + }, + { + name: 'Using Field Below', + value: 'fieldValue', + }, + ], + default: 'modelRequired', + }, + { + displayName: 'Value', + name: 'value', + type: 'string', + default: '', + hint: 'Use a {placeholder} for any data to be filled in by the model', + displayOptions: { + show: { + valueProvider: ['fieldValue'], + }, + }, + }, + ], + }, + ], +}; +export const placeholderDefinitionsCollection: INodeProperties = { + displayName: 'Placeholder Definitions', + name: 'placeholderDefinitions', + type: 'fixedCollection', + typeOptions: { + multipleValues: true, + }, + placeholder: 'Add Definition', + default: [], + options: [ + { + name: 'values', + displayName: 'Values', + values: [ + { + displayName: 'Placeholder Name', + name: 'name', + type: 'string', + default: '', + }, + { + displayName: 'Description', + name: 'description', + type: 'string', + default: '', + }, + { + displayName: 'Type', + name: 'type', + type: 'options', + // eslint-disable-next-line n8n-nodes-base/node-param-options-type-unsorted-items + options: [ + { + name: 'Not Specified (Default)', + value: 'not specified', + }, + { + name: 'String', + value: 'string', + }, + { + name: 'Number', + value: 'number', + }, + { + name: 'Boolean', + value: 'boolean', + }, + { + name: 'JSON', + value: 'json', + }, + ], + default: 'not specified', + }, + ], + }, + ], +}; + +export const jsonInput: INodeProperties = { + displayName: 'JSON', + name: 'json', + type: 'string', + typeOptions: { + rows: 5, + }, + hint: 'Use a {placeholder} for any data to be filled in by the model', + default: '', +}; + +export const authenticationProperties: INodeProperties[] = [ + { + displayName: 'Authentication', + name: 'authentication', + description: + 'Select the type of authentication to use if needed, authentication would be done by n8n and your credentials will not be shared with the LLM', + noDataExpression: true, + type: 'options', + options: [ + { + name: 'None', + value: 'none', + }, + { + name: 'Predefined Credential Type', + value: 'predefinedCredentialType', + description: + "We've already implemented auth for many services so that you don't have to set it up manually", + }, + { + name: 'Generic Credential Type', + value: 'genericCredentialType', + description: 'Fully customizable. Choose between basic, header, OAuth2, etc.', + }, + ], + default: 'none', + }, + { + displayName: 'Credential Type', + name: 'nodeCredentialType', + type: 'credentialsSelect', + noDataExpression: true, + required: true, + default: '', + credentialTypes: ['extends:oAuth2Api', 'extends:oAuth1Api', 'has:authenticate'], + displayOptions: { + show: { + authentication: ['predefinedCredentialType'], + }, + }, + }, + { + displayName: + 'Make sure you have specified the scope(s) for the Service Account in the credential', + name: 'googleApiWarning', + type: 'notice', + default: '', + displayOptions: { + show: { + nodeCredentialType: ['googleApi'], + }, + }, + }, + { + displayName: 'Generic Auth Type', + name: 'genericAuthType', + type: 'credentialsSelect', + required: true, + default: '', + credentialTypes: ['has:genericAuth'], + displayOptions: { + show: { + authentication: ['genericCredentialType'], + }, + }, + }, +]; + +export const optimizeResponseProperties: INodeProperties[] = [ + { + displayName: 'Optimize Response', + name: 'optimizeResponse', + type: 'boolean', + default: false, + noDataExpression: true, + description: + 'Whether the optimize the tool response to reduce amount of data passed to the LLM that could lead to better result and reduce cost', + }, + { + displayName: 'Expected Response Type', + name: 'responseType', + type: 'options', + displayOptions: { + show: { + optimizeResponse: [true], + }, + }, + options: [ + { + name: 'JSON', + value: 'json', + }, + { + name: 'HTML', + value: 'html', + }, + { + name: 'Text', + value: 'text', + }, + ], + default: 'json', + }, + { + displayName: 'Field Containing Data', + name: 'dataField', + type: 'string', + default: '', + placeholder: 'e.g. records', + description: 'Specify the name of the field in the response containing the data', + hint: 'leave blank to use whole response', + requiresDataPath: 'single', + displayOptions: { + show: { + optimizeResponse: [true], + responseType: ['json'], + }, + }, + }, + { + displayName: 'Include Fields', + name: 'fieldsToInclude', + type: 'options', + description: 'What fields response object should include', + default: 'all', + displayOptions: { + show: { + optimizeResponse: [true], + responseType: ['json'], + }, + }, + options: [ + { + name: 'All', + value: 'all', + description: 'Include all fields', + }, + { + name: 'Selected', + value: 'selected', + description: 'Include only fields specified below', + }, + { + name: 'Except', + value: 'except', + description: 'Exclude fields specified below', + }, + ], + }, + { + displayName: 'Fields', + name: 'fields', + type: 'string', + default: '', + placeholder: 'e.g. field1,field2', + description: + 'Comma-separated list of the field names. Supports dot notation. You can drag the selected fields from the input panel.', + requiresDataPath: 'multiple', + displayOptions: { + show: { + optimizeResponse: [true], + responseType: ['json'], + }, + hide: { + fieldsToInclude: ['all'], + }, + }, + }, + { + displayName: 'Selector (CSS)', + name: 'cssSelector', + type: 'string', + description: + 'Select specific element(e.g. body) or multiple elements(e.g. div) of chosen type in the response HTML.', + placeholder: 'e.g. body', + default: 'body', + displayOptions: { + show: { + optimizeResponse: [true], + responseType: ['html'], + }, + }, + }, + { + displayName: 'Return Only Content', + name: 'onlyContent', + type: 'boolean', + default: false, + description: + 'Whether to return only content of html elements, stripping html tags and attributes', + hint: 'Uses less tokens and may be easier for model to understand', + displayOptions: { + show: { + optimizeResponse: [true], + responseType: ['html'], + }, + }, + }, + { + displayName: 'Elements To Omit', + name: 'elementsToOmit', + type: 'string', + displayOptions: { + show: { + optimizeResponse: [true], + responseType: ['html'], + onlyContent: [true], + }, + }, + default: '', + placeholder: 'e.g. img, .className, #ItemId', + description: 'Comma-separated list of selectors that would be excluded when extracting content', + }, + { + displayName: 'Truncate Response', + name: 'truncateResponse', + type: 'boolean', + default: false, + hint: 'Helps save tokens', + displayOptions: { + show: { + optimizeResponse: [true], + responseType: ['text', 'html'], + }, + }, + }, + { + displayName: 'Max Response Characters', + name: 'maxLength', + type: 'number', + default: 1000, + typeOptions: { + minValue: 1, + }, + displayOptions: { + show: { + optimizeResponse: [true], + responseType: ['text', 'html'], + truncateResponse: [true], + }, + }, + }, +]; diff --git a/packages/@n8n/nodes-langchain/nodes/tools/ToolHttpRequest/httprequest.dark.svg b/packages/@n8n/nodes-langchain/nodes/tools/ToolHttpRequest/httprequest.dark.svg new file mode 100644 index 00000000000000..41d0b3204c1c74 --- /dev/null +++ b/packages/@n8n/nodes-langchain/nodes/tools/ToolHttpRequest/httprequest.dark.svg @@ -0,0 +1,3 @@ + + + diff --git a/packages/@n8n/nodes-langchain/nodes/tools/ToolHttpRequest/httprequest.svg b/packages/@n8n/nodes-langchain/nodes/tools/ToolHttpRequest/httprequest.svg new file mode 100644 index 00000000000000..955eb3940194af --- /dev/null +++ b/packages/@n8n/nodes-langchain/nodes/tools/ToolHttpRequest/httprequest.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/packages/@n8n/nodes-langchain/nodes/tools/ToolHttpRequest/interfaces.ts b/packages/@n8n/nodes-langchain/nodes/tools/ToolHttpRequest/interfaces.ts new file mode 100644 index 00000000000000..852d938cdce91b --- /dev/null +++ b/packages/@n8n/nodes-langchain/nodes/tools/ToolHttpRequest/interfaces.ts @@ -0,0 +1,23 @@ +export type ToolParameter = { + name: string; + required: boolean; + type?: string; + description?: string; + sendIn: SendIn; + key?: string; +}; + +export type PlaceholderDefinition = { + name: string; + type?: string; + description: string; +}; + +export type ParametersValues = Array<{ + name: string; + valueProvider: 'modelRequired' | 'modelOptional' | 'fieldValue'; + value?: string; +}>; + +export type ParameterInputType = 'keypair' | 'json' | 'model'; +export type SendIn = 'body' | 'qs' | 'path' | 'headers'; diff --git a/packages/@n8n/nodes-langchain/nodes/tools/ToolHttpRequest/utils.ts b/packages/@n8n/nodes-langchain/nodes/tools/ToolHttpRequest/utils.ts new file mode 100644 index 00000000000000..f1ec2e4864b546 --- /dev/null +++ b/packages/@n8n/nodes-langchain/nodes/tools/ToolHttpRequest/utils.ts @@ -0,0 +1,769 @@ +import type { + IExecuteFunctions, + IDataObject, + IHttpRequestOptions, + IRequestOptionsSimplified, + ExecutionError, + NodeApiError, +} from 'n8n-workflow'; +import { NodeConnectionType, NodeOperationError, jsonParse } from 'n8n-workflow'; + +import { getOAuth2AdditionalParameters } from 'n8n-nodes-base/dist/nodes/HttpRequest/GenericFunctions'; + +import type { + ParameterInputType, + ParametersValues, + PlaceholderDefinition, + ParametersValues as RawParametersValues, + SendIn, + ToolParameter, +} from './interfaces'; + +import set from 'lodash/set'; +import get from 'lodash/get'; +import unset from 'lodash/unset'; + +import cheerio from 'cheerio'; +import { convert } from 'html-to-text'; + +import { Readability } from '@mozilla/readability'; +import { JSDOM } from 'jsdom'; + +const genericCredentialRequest = async (ctx: IExecuteFunctions, itemIndex: number) => { + const genericType = ctx.getNodeParameter('genericAuthType', itemIndex) as string; + + if (genericType === 'httpBasicAuth' || genericType === 'httpDigestAuth') { + const basicAuth = await ctx.getCredentials('httpBasicAuth', itemIndex); + const sendImmediately = genericType === 'httpDigestAuth' ? false : undefined; + + return async (options: IHttpRequestOptions) => { + options.auth = { + username: basicAuth.user as string, + password: basicAuth.password as string, + sendImmediately, + }; + return await ctx.helpers.httpRequest(options); + }; + } + + if (genericType === 'httpHeaderAuth') { + const headerAuth = await ctx.getCredentials('httpHeaderAuth', itemIndex); + + return async (options: IHttpRequestOptions) => { + options.headers![headerAuth.name as string] = headerAuth.value; + return await ctx.helpers.httpRequest(options); + }; + } + + if (genericType === 'httpQueryAuth') { + const queryAuth = await ctx.getCredentials('httpQueryAuth', itemIndex); + + return async (options: IHttpRequestOptions) => { + if (!options.qs) { + options.qs = {}; + } + options.qs[queryAuth.name as string] = queryAuth.value; + return await ctx.helpers.httpRequest(options); + }; + } + + if (genericType === 'httpCustomAuth') { + const customAuth = await ctx.getCredentials('httpCustomAuth', itemIndex); + + return async (options: IHttpRequestOptions) => { + const auth = jsonParse((customAuth.json as string) || '{}', { + errorMessage: 'Invalid Custom Auth JSON', + }); + if (auth.headers) { + options.headers = { ...options.headers, ...auth.headers }; + } + if (auth.body) { + options.body = { ...(options.body as IDataObject), ...auth.body }; + } + if (auth.qs) { + options.qs = { ...options.qs, ...auth.qs }; + } + return await ctx.helpers.httpRequest(options); + }; + } + + if (genericType === 'oAuth1Api') { + return async (options: IHttpRequestOptions) => { + return await ctx.helpers.requestOAuth1.call(ctx, 'oAuth1Api', options); + }; + } + + if (genericType === 'oAuth2Api') { + return async (options: IHttpRequestOptions) => { + return await ctx.helpers.requestOAuth2.call(ctx, 'oAuth1Api', options, { + tokenType: 'Bearer', + }); + }; + } + + throw new NodeOperationError(ctx.getNode(), `The type ${genericType} is not supported`, { + itemIndex, + }); +}; + +const predefinedCredentialRequest = async (ctx: IExecuteFunctions, itemIndex: number) => { + const predefinedType = ctx.getNodeParameter('nodeCredentialType', itemIndex) as string; + const additionalOptions = getOAuth2AdditionalParameters(predefinedType); + + return async (options: IHttpRequestOptions) => { + return await ctx.helpers.requestWithAuthentication.call( + ctx, + predefinedType, + options, + additionalOptions && { oauth2: additionalOptions }, + itemIndex, + ); + }; +}; + +export const configureHttpRequestFunction = async ( + ctx: IExecuteFunctions, + credentialsType: 'predefinedCredentialType' | 'genericCredentialType' | 'none', + itemIndex: number, +) => { + switch (credentialsType) { + case 'genericCredentialType': + return await genericCredentialRequest(ctx, itemIndex); + case 'predefinedCredentialType': + return await predefinedCredentialRequest(ctx, itemIndex); + default: + return async (options: IHttpRequestOptions) => { + return await ctx.helpers.httpRequest(options); + }; + } +}; + +const defaultOptimizer = (response: T) => { + if (typeof response === 'string') { + return response; + } + if (typeof response === 'object') { + return JSON.stringify(response, null, 2); + } + + return String(response); +}; + +const htmlOptimizer = (ctx: IExecuteFunctions, itemIndex: number, maxLength: number) => { + const cssSelector = ctx.getNodeParameter('cssSelector', itemIndex, '') as string; + const onlyContent = ctx.getNodeParameter('onlyContent', itemIndex, false) as boolean; + let elementsToOmit: string[] = []; + + if (onlyContent) { + const elementsToOmitUi = ctx.getNodeParameter('elementsToOmit', itemIndex, '') as + | string + | string[]; + + if (typeof elementsToOmitUi === 'string') { + elementsToOmit = elementsToOmitUi + .split(',') + .filter((s) => s) + .map((s) => s.trim()); + } + } + + return (response: T) => { + if (typeof response !== 'string') { + throw new NodeOperationError( + ctx.getNode(), + `The response type must be a string. Received: ${typeof response}`, + { itemIndex }, + ); + } + const returnData: string[] = []; + const html = cheerio.load(response); + const htmlElements = html(cssSelector); + + htmlElements.each((_, el) => { + let value = html(el).html() || ''; + + if (onlyContent) { + let htmlToTextOptions; + + if (elementsToOmit?.length) { + htmlToTextOptions = { + selectors: elementsToOmit.map((selector) => ({ + selector, + format: 'skip', + })), + }; + } + + value = convert(value, htmlToTextOptions); + } + + value = value + .trim() + .replace(/^\s+|\s+$/g, '') + .replace(/(\r\n|\n|\r)/gm, '') + .replace(/\s+/g, ' '); + + returnData.push(value); + }); + + const text = JSON.stringify(returnData, null, 2); + + if (maxLength > 0 && text.length > maxLength) { + return text.substring(0, maxLength); + } + + return text; + }; +}; + +const textOptimizer = (ctx: IExecuteFunctions, itemIndex: number, maxLength: number) => { + return (response: string | IDataObject) => { + if (typeof response === 'object') { + try { + response = JSON.stringify(response, null, 2); + } catch (error) {} + } + + if (typeof response !== 'string') { + throw new NodeOperationError( + ctx.getNode(), + `The response type must be a string. Received: ${typeof response}`, + { itemIndex }, + ); + } + + const dom = new JSDOM(response); + const article = new Readability(dom.window.document, { + keepClasses: true, + }).parse(); + + const text = article?.textContent || ''; + + if (maxLength > 0 && text.length > maxLength) { + return text.substring(0, maxLength); + } + + return text; + }; +}; + +const jsonOptimizer = (ctx: IExecuteFunctions, itemIndex: number) => { + return (response: string): string => { + let responseData: IDataObject | IDataObject[] | string = response; + + if (typeof responseData === 'string') { + responseData = jsonParse(response); + } + + if (typeof responseData !== 'object' || !responseData) { + throw new NodeOperationError( + ctx.getNode(), + 'The response type must be an object or an array of objects', + { itemIndex }, + ); + } + + const dataField = ctx.getNodeParameter('dataField', itemIndex, '') as string; + let returnData: IDataObject[] = []; + + if (!Array.isArray(responseData)) { + if (dataField) { + const data = responseData[dataField] as IDataObject | IDataObject[]; + if (Array.isArray(data)) { + responseData = data; + } else { + responseData = [data]; + } + } else { + responseData = [responseData]; + } + } else { + if (dataField) { + responseData = responseData.map((data) => data[dataField]) as IDataObject[]; + } + } + + const fieldsToInclude = ctx.getNodeParameter('fieldsToInclude', itemIndex, 'all') as + | 'all' + | 'selected' + | 'except'; + + let fields: string | string[] = []; + + if (fieldsToInclude !== 'all') { + fields = ctx.getNodeParameter('fields', itemIndex, []) as string[] | string; + + if (typeof fields === 'string') { + fields = fields.split(',').map((field) => field.trim()); + } + } else { + returnData = responseData; + } + + if (fieldsToInclude === 'selected') { + for (const item of responseData) { + const newItem: IDataObject = {}; + + for (const field of fields) { + set(newItem, field, get(item, field)); + } + + returnData.push(newItem); + } + } + + if (fieldsToInclude === 'except') { + for (const item of responseData) { + for (const field of fields) { + unset(item, field); + } + + returnData.push(item); + } + } + + return JSON.stringify(returnData, null, 2); + }; +}; + +export const configureResponseOptimizer = (ctx: IExecuteFunctions, itemIndex: number) => { + const optimizeResponse = ctx.getNodeParameter('optimizeResponse', itemIndex, false) as boolean; + + if (optimizeResponse) { + const responseType = ctx.getNodeParameter('responseType', itemIndex) as + | 'json' + | 'text' + | 'html'; + + let maxLength = 0; + const truncateResponse = ctx.getNodeParameter('truncateResponse', itemIndex, false) as boolean; + + if (truncateResponse) { + maxLength = ctx.getNodeParameter('maxLength', itemIndex, 0) as number; + } + + switch (responseType) { + case 'html': + return htmlOptimizer(ctx, itemIndex, maxLength); + case 'text': + return textOptimizer(ctx, itemIndex, maxLength); + case 'json': + return jsonOptimizer(ctx, itemIndex); + } + } + + return defaultOptimizer; +}; + +const extractPlaceholders = (text: string): string[] => { + const placeholder = /(\{[a-zA-Z0-9_]+\})/g; + const returnData: string[] = []; + + const matches = text.matchAll(placeholder); + + for (const match of matches) { + returnData.push(match[0].replace(/{|}/g, '')); + } + + return returnData; +}; + +export const extractParametersFromText = ( + placeholders: PlaceholderDefinition[], + text: string, + sendIn: SendIn, + key?: string, +): ToolParameter[] => { + if (typeof text !== 'string') return []; + + const parameters = extractPlaceholders(text); + + if (parameters.length) { + // eslint-disable-next-line @typescript-eslint/no-use-before-define + const inputParameters = prepareParameters( + parameters.map((name) => ({ + name, + valueProvider: 'modelRequired', + })), + placeholders, + 'keypair', + sendIn, + '', + ); + + return key + ? inputParameters.parameters.map((p) => ({ ...p, key })) + : inputParameters.parameters; + } + + return []; +}; + +function prepareParameters( + rawParameters: RawParametersValues, + placeholders: PlaceholderDefinition[], + parametersInputType: 'model' | 'keypair' | 'json', + sendIn: SendIn, + modelInputDescription: string, + jsonWithPlaceholders?: string, +): { parameters: ToolParameter[]; values: IDataObject } { + const parameters: ToolParameter[] = []; + const values: IDataObject = {}; + + if (parametersInputType === 'model') { + return { + parameters: [ + { + name: sendIn, + required: true, + type: 'json', + description: modelInputDescription, + sendIn, + }, + ], + values: {}, + }; + } + + if (parametersInputType === 'keypair') { + for (const entry of rawParameters) { + if (entry.valueProvider.includes('model')) { + const placeholder = placeholders.find((p) => p.name === entry.name); + + const parameter: ToolParameter = { + name: entry.name, + required: entry.valueProvider === 'modelRequired', + sendIn, + }; + + if (placeholder) { + parameter.type = placeholder.type; + parameter.description = placeholder.description; + } + + parameters.push(parameter); + } else if (entry.value) { + // if value has placeholders push them to parameters + parameters.push( + ...extractParametersFromText(placeholders, entry.value, sendIn, entry.name), + ); + values[entry.name] = entry.value; //push to user provided values + } + } + } + + if (parametersInputType === 'json' && jsonWithPlaceholders) { + parameters.push( + ...extractParametersFromText(placeholders, jsonWithPlaceholders, sendIn, `${sendIn + 'Raw'}`), + ); + } + + return { + parameters, + values, + }; +} + +const MODEL_INPUT_DESCRIPTION = { + qs: 'Query parameters for request as key value pairs', + headers: 'Headers parameters for request as key value pairs', + body: 'Body parameters for request as key value pairs', +}; + +export const updateParametersAndOptions = (options: { + ctx: IExecuteFunctions; + itemIndex: number; + toolParameters: ToolParameter[]; + placeholdersDefinitions: PlaceholderDefinition[]; + requestOptions: IHttpRequestOptions; + rawRequestOptions: { [key: string]: string }; + requestOptionsProperty: 'headers' | 'qs' | 'body'; + inputTypePropertyName: string; + jsonPropertyName: string; + parametersPropertyName: string; +}) => { + const { + ctx, + itemIndex, + toolParameters, + placeholdersDefinitions, + requestOptions, + rawRequestOptions, + requestOptionsProperty, + inputTypePropertyName, + jsonPropertyName, + parametersPropertyName, + } = options; + + const inputType = ctx.getNodeParameter( + inputTypePropertyName, + itemIndex, + 'keypair', + ) as ParameterInputType; + + let parametersValues: ParametersValues = []; + + if (inputType === 'json') { + rawRequestOptions[requestOptionsProperty] = ctx.getNodeParameter( + jsonPropertyName, + itemIndex, + '', + ) as string; + } else { + parametersValues = ctx.getNodeParameter( + parametersPropertyName, + itemIndex, + [], + ) as ParametersValues; + } + + const inputParameters = prepareParameters( + parametersValues, + placeholdersDefinitions, + inputType, + requestOptionsProperty, + MODEL_INPUT_DESCRIPTION[requestOptionsProperty], + rawRequestOptions[requestOptionsProperty], + ); + + toolParameters.push(...inputParameters.parameters); + + requestOptions[requestOptionsProperty] = { + ...(requestOptions[requestOptionsProperty] as IDataObject), + ...inputParameters.values, + }; +}; + +const getParametersDescription = (parameters: ToolParameter[]) => + parameters + .map( + (p) => + `${p.name}: (description: ${p.description ?? ''}, type: ${p.type ?? 'string'}, required: ${!!p.required})`, + ) + .join(',\n '); + +export const prepareToolDescription = ( + toolDescription: string, + toolParameters: ToolParameter[], +) => { + let description = `${toolDescription}`; + + if (toolParameters.length) { + description += ` + Tool expects valid stringified JSON object with ${toolParameters.length} properties. + Property names with description, type and required status: + ${getParametersDescription(toolParameters)} + ALL parameters marked as required must be provided`; + } + + return description; +}; + +export const configureToolFunction = ( + ctx: IExecuteFunctions, + itemIndex: number, + toolParameters: ToolParameter[], + requestOptions: IHttpRequestOptions, + rawRequestOptions: { [key: string]: string }, + httpRequest: (options: IHttpRequestOptions) => Promise, + optimizeResponse: (response: string) => string, +) => { + return async (query: string): Promise => { + const { index } = ctx.addInputData(NodeConnectionType.AiTool, [[{ json: { query } }]]); + + let response: string = ''; + let options: IHttpRequestOptions | null = null; + let executionError: Error | undefined = undefined; + + if (!toolParameters.length) { + query = '{}'; + } + + try { + if (query) { + let dataFromModel; + + try { + dataFromModel = jsonParse(query); + } catch (error) { + if (toolParameters.length === 1) { + dataFromModel = { [toolParameters[0].name]: query }; + } else { + throw new NodeOperationError( + ctx.getNode(), + `Input is not a valid JSON: ${error.message}`, + { itemIndex }, + ); + } + } + + for (const parameter of toolParameters) { + if ( + parameter.required && + (dataFromModel[parameter.name] === undefined || dataFromModel[parameter.name] === null) + ) { + throw new NodeOperationError( + ctx.getNode(), + `Model did not provide parameter '${parameter.name}' which is required and must be present in the input`, + { itemIndex }, + ); + } + } + + options = requestOptions; + + for (const parameter of toolParameters) { + let argument = dataFromModel[parameter.name]; + + if ( + argument && + parameter.type === 'json' && + !['qsRaw', 'headersRaw', 'bodyRaw'].includes(parameter.key ?? '') && + typeof argument !== 'object' + ) { + try { + argument = jsonParse(String(argument)); + } catch (error) { + throw new NodeOperationError( + ctx.getNode(), + `Parameter ${parameter.name} is not a valid JSON: ${error.message}`, + { + itemIndex, + }, + ); + } + } + + if (parameter.sendIn === 'path') { + argument = String(argument); + + //remove " or ' from start or end + argument = argument.replace(/^['"]+|['"]+$/g, ''); + + options.url = options.url.replace(`{${parameter.name}}`, argument); + continue; + } + + if (parameter.sendIn === parameter.name) { + set(options, [parameter.sendIn], argument); + continue; + } + + if (['qsRaw', 'headersRaw', 'bodyRaw'].includes(parameter.key ?? '')) { + //enclose string in quotes as user and model could omit them + if (parameter.type === 'string') { + argument = String(argument); + if ( + !argument.startsWith('"') && + !rawRequestOptions[parameter.sendIn].includes(`"{${parameter.name}}"`) + ) { + argument = `"${argument}"`; + } + } + + if (typeof argument === 'object') { + argument = JSON.stringify(argument); + } + + rawRequestOptions[parameter.sendIn] = rawRequestOptions[parameter.sendIn].replace( + `{${parameter.name}}`, + String(argument), + ); + continue; + } + + if (parameter.key) { + let requestOptionsValue = get(options, [parameter.sendIn, parameter.key]); + + if (typeof requestOptionsValue === 'string') { + requestOptionsValue = requestOptionsValue.replace( + `{${parameter.name}}`, + String(argument), + ); + } + + set(options, [parameter.sendIn, parameter.key], requestOptionsValue); + continue; + } + + set(options, [parameter.sendIn, parameter.name], argument); + } + + for (const [key, value] of Object.entries(rawRequestOptions)) { + if (value) { + let parsedValue; + try { + parsedValue = jsonParse(value); + } catch (error) { + let recoveredData = ''; + try { + recoveredData = value + .replace(/'/g, '"') // Replace single quotes with double quotes + .replace(/(['"])?([a-zA-Z0-9_]+)(['"])?:/g, '"$2":') // Wrap keys in double quotes + .replace(/,\s*([\]}])/g, '$1') // Remove trailing commas from objects + .replace(/,+$/, ''); // Remove trailing comma + parsedValue = jsonParse(recoveredData); + } catch (err) { + throw new NodeOperationError( + ctx.getNode(), + `Could not replace placeholders in ${key}: ${error.message}`, + ); + } + } + options[key as 'qs' | 'headers' | 'body'] = parsedValue; + } + } + } + + if (options) { + options.url = encodeURI(options.url); + + if (options.headers && !Object.keys(options.headers).length) { + delete options.headers; + } + if (options.qs && !Object.keys(options.qs).length) { + delete options.qs; + } + if (options.body && !Object.keys(options.body).length) { + delete options.body; + } + } + } catch (error) { + const errorMessage = 'Input provided by model is not valid'; + + if (error instanceof NodeOperationError) { + executionError = error; + } else { + executionError = new NodeOperationError(ctx.getNode(), errorMessage, { + itemIndex, + }); + } + + response = errorMessage; + } + + if (options) { + try { + response = optimizeResponse(await httpRequest(options)); + } catch (error) { + const httpCode = (error as NodeApiError).httpCode; + response = `${httpCode ? `HTTP ${httpCode} ` : ''}There was an error: "${error.message}"`; + } + } + + if (typeof response !== 'string') { + executionError = new NodeOperationError(ctx.getNode(), 'Wrong output type returned', { + description: `The response property should be a string, but it is an ${typeof response}`, + }); + response = `There was an error: "${executionError.message}"`; + } + + if (executionError) { + void ctx.addOutputData(NodeConnectionType.AiTool, index, executionError as ExecutionError); + } else { + void ctx.addOutputData(NodeConnectionType.AiTool, index, [[{ json: { response } }]]); + } + + return response; + }; +}; diff --git a/packages/@n8n/nodes-langchain/package.json b/packages/@n8n/nodes-langchain/package.json index 725f25df3c454a..4e52a1cecc9feb 100644 --- a/packages/@n8n/nodes-langchain/package.json +++ b/packages/@n8n/nodes-langchain/package.json @@ -98,6 +98,7 @@ "dist/nodes/text_splitters/TextSplitterTokenSplitter/TextSplitterTokenSplitter.node.js", "dist/nodes/tools/ToolCalculator/ToolCalculator.node.js", "dist/nodes/tools/ToolCode/ToolCode.node.js", + "dist/nodes/tools/ToolHttpRequest/ToolHttpRequest.node.js", "dist/nodes/tools/ToolSerpApi/ToolSerpApi.node.js", "dist/nodes/tools/ToolWikipedia/ToolWikipedia.node.js", "dist/nodes/tools/ToolWolframAlpha/ToolWolframAlpha.node.js", @@ -122,6 +123,7 @@ "devDependencies": { "@aws-sdk/types": "^3.535.0", "@types/basic-auth": "^1.1.3", + "@types/cheerio": "^0.22.15", "@types/express": "^4.17.21", "@types/html-to-text": "^9.0.1", "@types/json-schema": "^7.0.15", @@ -146,6 +148,7 @@ "@langchain/pinecone": "0.0.6", "@langchain/redis": "0.0.5", "@langchain/textsplitters": "0.0.2", + "@mozilla/readability": "^0.5.0", "@n8n/typeorm": "0.3.20-10", "@n8n/vm2": "3.9.20", "@pinecone-database/pinecone": "2.2.1", @@ -153,12 +156,14 @@ "@supabase/supabase-js": "2.43.4", "@xata.io/client": "0.28.4", "basic-auth": "2.0.1", + "cheerio": "1.0.0-rc.12", "cohere-ai": "7.10.1", "d3-dsv": "2.0.0", "epub2": "3.0.2", "form-data": "4.0.0", "generate-schema": "2.6.0", "html-to-text": "9.0.5", + "jsdom": "^23.0.1", "json-schema-to-zod": "2.1.0", "langchain": "0.2.2", "lodash": "4.17.21", diff --git a/packages/core/src/Constants.ts b/packages/core/src/Constants.ts index dd6277b69bdd24..d5fc12bffd983b 100644 --- a/packages/core/src/Constants.ts +++ b/packages/core/src/Constants.ts @@ -2,6 +2,7 @@ export const CUSTOM_EXTENSION_ENV = 'N8N_CUSTOM_EXTENSIONS'; export const PLACEHOLDER_EMPTY_EXECUTION_ID = '__UNKNOWN__'; export const PLACEHOLDER_EMPTY_WORKFLOW_ID = '__EMPTY__'; export const HTTP_REQUEST_NODE_TYPE = 'n8n-nodes-base.httpRequest'; +export const HTTP_REQUEST_TOOL_NODE_TYPE = '@n8n/n8n-nodes-langchain.toolHttpRequest'; export const CUSTOM_NODES_CATEGORY = 'Custom Nodes'; diff --git a/packages/core/src/NodeExecuteFunctions.ts b/packages/core/src/NodeExecuteFunctions.ts index e808224864ac6a..d859a4e2e8ee5a 100644 --- a/packages/core/src/NodeExecuteFunctions.ts +++ b/packages/core/src/NodeExecuteFunctions.ts @@ -136,6 +136,7 @@ import { CONFIG_FILES, CUSTOM_EXTENSION_ENV, HTTP_REQUEST_NODE_TYPE, + HTTP_REQUEST_TOOL_NODE_TYPE, PLACEHOLDER_EMPTY_EXECUTION_ID, RESTRICT_FILE_ACCESS_TO, UM_EMAIL_TEMPLATES_INVITE, @@ -1997,7 +1998,7 @@ export async function getCredentials( // Hardcode for now for security reasons that only a single node can access // all credentials - const fullAccess = [HTTP_REQUEST_NODE_TYPE].includes(node.type); + const fullAccess = [HTTP_REQUEST_NODE_TYPE, HTTP_REQUEST_TOOL_NODE_TYPE].includes(node.type); let nodeCredentialDescription: INodeCredentialDescription | undefined; if (!fullAccess) { diff --git a/packages/nodes-base/nodes/Splunk/splunk.svg b/packages/nodes-base/nodes/Splunk/splunk.svg index 8eb25b4a26ff0b..d922ab4151f33e 100644 --- a/packages/nodes-base/nodes/Splunk/splunk.svg +++ b/packages/nodes-base/nodes/Splunk/splunk.svg @@ -1,134 +1,633 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/packages/workflow/src/Constants.ts b/packages/workflow/src/Constants.ts index 8a7494980cae66..be9a323e6beba3 100644 --- a/packages/workflow/src/Constants.ts +++ b/packages/workflow/src/Constants.ts @@ -59,10 +59,12 @@ export const CHAIN_SUMMARIZATION_LANGCHAIN_NODE_TYPE = '@n8n/n8n-nodes-langchain.chainSummarization'; export const CODE_TOOL_LANGCHAIN_NODE_TYPE = '@n8n/n8n-nodes-langchain.toolCode'; export const WORKFLOW_TOOL_LANGCHAIN_NODE_TYPE = '@n8n/n8n-nodes-langchain.toolWorkflow'; +export const HTTP_REQUEST_TOOL_LANGCHAIN_NODE_TYPE = '@n8n/n8n-nodes-langchain.toolHttpRequest'; export const LANGCHAIN_CUSTOM_TOOLS = [ CODE_TOOL_LANGCHAIN_NODE_TYPE, WORKFLOW_TOOL_LANGCHAIN_NODE_TYPE, + HTTP_REQUEST_TOOL_LANGCHAIN_NODE_TYPE, ]; //nodes that would execute only once with such parameters diff --git a/packages/workflow/src/Interfaces.ts b/packages/workflow/src/Interfaces.ts index 52b5e391dfd9a5..7e555c911447bd 100644 --- a/packages/workflow/src/Interfaces.ts +++ b/packages/workflow/src/Interfaces.ts @@ -2314,6 +2314,7 @@ export interface INodeGraphItem { src_instance_id?: string; agent?: string; //@n8n/n8n-nodes-langchain.agent prompts?: IDataObject[] | IDataObject; //ai node's prompts, cloud only + toolSettings?: IDataObject; //various langchain tool's settings } export interface INodeNameIndex { diff --git a/packages/workflow/src/TelemetryHelpers.ts b/packages/workflow/src/TelemetryHelpers.ts index f2f51e9c7dfe9f..3113a91554c2d0 100644 --- a/packages/workflow/src/TelemetryHelpers.ts +++ b/packages/workflow/src/TelemetryHelpers.ts @@ -16,6 +16,7 @@ import { CHAIN_LLM_LANGCHAIN_NODE_TYPE, CHAIN_SUMMARIZATION_LANGCHAIN_NODE_TYPE, HTTP_REQUEST_NODE_TYPE, + HTTP_REQUEST_TOOL_LANGCHAIN_NODE_TYPE, LANGCHAIN_CUSTOM_TOOLS, OPENAI_LANGCHAIN_NODE_TYPE, STICKY_NODE_TYPE, @@ -30,6 +31,34 @@ export function isNumber(value: unknown): value is number { return typeof value === 'number'; } +const countPlaceholders = (text: string) => { + const placeholder = /(\{[a-zA-Z0-9_]+\})/g; + let returnData = 0; + + try { + const matches = text.matchAll(placeholder); + for (const _ of matches) returnData++; + } catch (error) {} + + return returnData; +}; + +const countPlaceholdersInParameters = (parameters: IDataObject[]) => { + let returnData = 0; + + for (const parameter of parameters) { + if (!parameter.value) { + //count parameters provided by model + returnData++; + } else { + //check if any placeholders in user provided value + returnData += countPlaceholders(String(parameter.value)); + } + } + + return returnData; +}; + type XYPosition = [number, number]; function areOverlapping( @@ -199,6 +228,87 @@ export function generateNodesGraph( nodeItem.domain_base = getDomainBase(url); nodeItem.domain_path = getDomainPath(url); nodeItem.method = node.parameters.requestMethod as string; + } else if (HTTP_REQUEST_TOOL_LANGCHAIN_NODE_TYPE === node.type) { + if (!nodeItem.toolSettings) nodeItem.toolSettings = {}; + + nodeItem.toolSettings.url_type = 'other'; + nodeItem.toolSettings.uses_auth = false; + nodeItem.toolSettings.placeholders = 0; + nodeItem.toolSettings.query_from_model_only = false; + nodeItem.toolSettings.headers_from_model_only = false; + nodeItem.toolSettings.body_from_model_only = false; + + const toolUrl = (node.parameters?.url as string) ?? ''; + nodeItem.toolSettings.placeholders += countPlaceholders(toolUrl); + + const authType = (node.parameters?.authentication as string) ?? ''; + + if (authType && authType !== 'none') { + nodeItem.toolSettings.uses_auth = true; + } + + if (toolUrl.startsWith('{') && toolUrl.endsWith('}')) { + nodeItem.toolSettings.url_type = 'any'; + } else if (toolUrl.includes('google.com')) { + nodeItem.toolSettings.url_type = 'google'; + } + + if (node.parameters?.sendBody) { + if (node.parameters?.specifyBody === 'model') { + nodeItem.toolSettings.body_from_model_only = true; + } + + if (node.parameters?.jsonBody) { + nodeItem.toolSettings.placeholders += countPlaceholders( + node.parameters?.jsonBody as string, + ); + } + + if (node.parameters?.parametersBody) { + const parameters = (node.parameters?.parametersBody as IDataObject) + .values as IDataObject[]; + + nodeItem.toolSettings.placeholders += countPlaceholdersInParameters(parameters); + } + } + + if (node.parameters?.sendHeaders) { + if (node.parameters?.specifyHeaders === 'model') { + nodeItem.toolSettings.headers_from_model_only = true; + } + + if (node.parameters?.jsonHeaders) { + nodeItem.toolSettings.placeholders += countPlaceholders( + node.parameters?.jsonHeaders as string, + ); + } + + if (node.parameters?.parametersHeaders) { + const parameters = (node.parameters?.parametersHeaders as IDataObject) + .values as IDataObject[]; + + nodeItem.toolSettings.placeholders += countPlaceholdersInParameters(parameters); + } + } + + if (node.parameters?.sendQuery) { + if (node.parameters?.specifyQuery === 'model') { + nodeItem.toolSettings.query_from_model_only = true; + } + + if (node.parameters?.jsonQuery) { + nodeItem.toolSettings.placeholders += countPlaceholders( + node.parameters?.jsonQuery as string, + ); + } + + if (node.parameters?.parametersQuery) { + const parameters = (node.parameters?.parametersQuery as IDataObject) + .values as IDataObject[]; + + nodeItem.toolSettings.placeholders += countPlaceholdersInParameters(parameters); + } + } } else if (node.type === WEBHOOK_NODE_TYPE) { webhookNodeNames.push(node.name); } else { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 249f549b1217cb..95e37deb5d7454 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -275,7 +275,7 @@ importers: version: 0.0.10(encoding@0.1.13) '@langchain/community': specifier: 0.2.2 - version: 0.2.2(@aws-sdk/client-bedrock-runtime@3.535.0)(@aws-sdk/client-s3@3.478.0)(@aws-sdk/credential-provider-node@3.535.0)(@getzep/zep-js@0.9.0)(@google-ai/generativelanguage@2.5.0(encoding@0.1.13))(@google-cloud/storage@6.11.0(encoding@0.1.13))(@huggingface/inference@2.7.0)(@pinecone-database/pinecone@2.2.1)(@qdrant/js-client-rest@1.9.0(typescript@5.4.2))(@smithy/eventstream-codec@2.2.0)(@smithy/protocol-http@3.3.0)(@smithy/signature-v4@2.2.1)(@smithy/util-utf8@2.3.0)(@supabase/postgrest-js@1.15.2)(@supabase/supabase-js@2.43.4)(@xata.io/client@0.28.4(typescript@5.4.2))(axios@1.6.7)(cohere-ai@7.10.1(encoding@0.1.13))(d3-dsv@2.0.0)(encoding@0.1.13)(epub2@3.0.2(ts-toolbelt@9.6.0))(fast-xml-parser@4.3.5)(handlebars@4.7.8)(html-to-text@9.0.5)(ignore@5.2.4)(ioredis@5.3.2)(jsdom@23.0.1)(jsonwebtoken@9.0.2)(lodash@4.17.21)(mammoth@1.7.2)(mysql2@3.10.0)(pdf-parse@1.1.1)(pg@8.11.3)(redis@4.6.12)(ws@8.14.2) + version: 0.2.2(@aws-sdk/client-bedrock-runtime@3.535.0)(@aws-sdk/client-s3@3.478.0)(@aws-sdk/credential-provider-node@3.535.0)(@getzep/zep-js@0.9.0)(@google-ai/generativelanguage@2.5.0(encoding@0.1.13))(@google-cloud/storage@6.11.0(encoding@0.1.13))(@huggingface/inference@2.7.0)(@mozilla/readability@0.5.0)(@pinecone-database/pinecone@2.2.1)(@qdrant/js-client-rest@1.9.0(typescript@5.4.2))(@smithy/eventstream-codec@2.2.0)(@smithy/protocol-http@3.3.0)(@smithy/signature-v4@2.2.1)(@smithy/util-utf8@2.3.0)(@supabase/postgrest-js@1.15.2)(@supabase/supabase-js@2.43.4)(@xata.io/client@0.28.4(typescript@5.4.2))(axios@1.6.7)(cheerio@1.0.0-rc.12)(cohere-ai@7.10.1(encoding@0.1.13))(d3-dsv@2.0.0)(encoding@0.1.13)(epub2@3.0.2(ts-toolbelt@9.6.0))(fast-xml-parser@4.3.5)(handlebars@4.7.8)(html-to-text@9.0.5)(ignore@5.2.4)(ioredis@5.3.2)(jsdom@23.0.1)(jsonwebtoken@9.0.2)(lodash@4.17.21)(mammoth@1.7.2)(mysql2@3.10.0)(pdf-parse@1.1.1)(pg@8.11.3)(redis@4.6.12)(ws@8.14.2) '@langchain/core': specifier: 0.2.0 version: 0.2.0 @@ -300,6 +300,9 @@ importers: '@langchain/textsplitters': specifier: 0.0.2 version: 0.0.2 + '@mozilla/readability': + specifier: ^0.5.0 + version: 0.5.0 '@n8n/typeorm': specifier: 0.3.20-10 version: 0.3.20-10(@sentry/node@7.87.0)(ioredis@5.3.2)(mssql@10.0.2)(mysql2@3.10.0)(pg@8.11.3)(redis@4.6.12)(sqlite3@5.1.7) @@ -321,6 +324,9 @@ importers: basic-auth: specifier: 2.0.1 version: 2.0.1 + cheerio: + specifier: 1.0.0-rc.12 + version: 1.0.0-rc.12 cohere-ai: specifier: 7.10.1 version: 7.10.1(encoding@0.1.13) @@ -339,12 +345,15 @@ importers: html-to-text: specifier: 9.0.5 version: 9.0.5 + jsdom: + specifier: ^23.0.1 + version: 23.0.1 json-schema-to-zod: specifier: 2.1.0 version: 2.1.0 langchain: specifier: 0.2.2 - version: 0.2.2(@aws-sdk/client-s3@3.478.0)(@aws-sdk/credential-provider-node@3.535.0)(@google-ai/generativelanguage@2.5.0(encoding@0.1.13))(@pinecone-database/pinecone@2.2.1)(@supabase/supabase-js@2.43.4)(@xata.io/client@0.28.4(typescript@5.4.2))(axios@1.6.7)(d3-dsv@2.0.0)(encoding@0.1.13)(epub2@3.0.2(ts-toolbelt@9.6.0))(fast-xml-parser@4.3.5)(handlebars@4.7.8)(html-to-text@9.0.5)(ignore@5.2.4)(ioredis@5.3.2)(jsdom@23.0.1)(mammoth@1.7.2)(pdf-parse@1.1.1)(redis@4.6.12)(ws@8.14.2) + version: 0.2.2(@aws-sdk/client-s3@3.478.0)(@aws-sdk/credential-provider-node@3.535.0)(@google-ai/generativelanguage@2.5.0(encoding@0.1.13))(@pinecone-database/pinecone@2.2.1)(@supabase/supabase-js@2.43.4)(@xata.io/client@0.28.4(typescript@5.4.2))(axios@1.6.7)(cheerio@1.0.0-rc.12)(d3-dsv@2.0.0)(encoding@0.1.13)(epub2@3.0.2(ts-toolbelt@9.6.0))(fast-xml-parser@4.3.5)(handlebars@4.7.8)(html-to-text@9.0.5)(ignore@5.2.4)(ioredis@5.3.2)(jsdom@23.0.1)(mammoth@1.7.2)(pdf-parse@1.1.1)(redis@4.6.12)(ws@8.14.2) lodash: specifier: 4.17.21 version: 4.17.21 @@ -391,6 +400,9 @@ importers: '@types/basic-auth': specifier: ^1.1.3 version: 1.1.3 + '@types/cheerio': + specifier: ^0.22.15 + version: 0.22.31 '@types/express': specifier: ^4.17.21 version: 4.17.21 @@ -512,7 +524,7 @@ importers: dependencies: '@langchain/community': specifier: 0.2.2 - version: 0.2.2(@aws-sdk/client-bedrock-runtime@3.535.0)(@aws-sdk/client-s3@3.478.0)(@aws-sdk/credential-provider-node@3.535.0)(@getzep/zep-js@0.9.0)(@google-cloud/storage@6.11.0(encoding@0.1.13))(@huggingface/inference@2.7.0)(@pinecone-database/pinecone@2.1.0)(@qdrant/js-client-rest@1.9.0(typescript@5.4.2))(@smithy/eventstream-codec@2.2.0)(@smithy/protocol-http@3.3.0)(@smithy/signature-v4@2.2.1)(@smithy/util-utf8@2.3.0)(@supabase/postgrest-js@1.15.2)(@supabase/supabase-js@2.43.4)(@xata.io/client@0.28.4(typescript@5.4.2))(axios@1.6.7)(cohere-ai@7.10.1(encoding@0.1.13))(d3-dsv@2.0.0)(encoding@0.1.13)(epub2@3.0.2)(fast-xml-parser@4.3.5)(handlebars@4.7.8)(html-to-text@9.0.5)(ignore@5.2.4)(ioredis@5.3.2)(jsdom@23.0.1)(jsonwebtoken@9.0.2)(lodash@4.17.21)(mammoth@1.7.2)(mysql2@3.10.0)(pdf-parse@1.1.1)(pg@8.11.3)(redis@4.6.13)(ws@8.14.2) + version: 0.2.2(@aws-sdk/client-bedrock-runtime@3.535.0)(@aws-sdk/client-s3@3.478.0)(@aws-sdk/credential-provider-node@3.535.0)(@getzep/zep-js@0.9.0)(@google-cloud/storage@6.11.0(encoding@0.1.13))(@huggingface/inference@2.7.0)(@mozilla/readability@0.5.0)(@pinecone-database/pinecone@2.1.0)(@qdrant/js-client-rest@1.9.0(typescript@5.4.2))(@smithy/eventstream-codec@2.2.0)(@smithy/protocol-http@3.3.0)(@smithy/signature-v4@2.2.1)(@smithy/util-utf8@2.3.0)(@supabase/postgrest-js@1.15.2)(@supabase/supabase-js@2.43.4)(@xata.io/client@0.28.4(typescript@5.4.2))(axios@1.6.7)(cheerio@1.0.0-rc.12)(cohere-ai@7.10.1(encoding@0.1.13))(d3-dsv@2.0.0)(encoding@0.1.13)(epub2@3.0.2)(fast-xml-parser@4.3.5)(handlebars@4.7.8)(html-to-text@9.0.5)(ignore@5.2.4)(ioredis@5.3.2)(jsdom@23.0.1)(jsonwebtoken@9.0.2)(lodash@4.17.21)(mammoth@1.7.2)(mysql2@3.10.0)(pdf-parse@1.1.1)(pg@8.11.3)(redis@4.6.13)(ws@8.14.2) '@langchain/core': specifier: 0.2.0 version: 0.2.0 @@ -665,7 +677,7 @@ importers: version: 9.0.2 langchain: specifier: 0.2.2 - version: 0.2.2(@aws-sdk/client-s3@3.478.0)(@aws-sdk/credential-provider-node@3.535.0)(@pinecone-database/pinecone@2.1.0)(@supabase/supabase-js@2.43.4)(@xata.io/client@0.28.4(typescript@5.4.2))(axios@1.6.7)(d3-dsv@2.0.0)(encoding@0.1.13)(epub2@3.0.2)(fast-xml-parser@4.3.5)(handlebars@4.7.8)(html-to-text@9.0.5)(ignore@5.2.4)(ioredis@5.3.2)(jsdom@23.0.1)(mammoth@1.7.2)(pdf-parse@1.1.1)(redis@4.6.13)(ws@8.14.2) + version: 0.2.2(@aws-sdk/client-s3@3.478.0)(@aws-sdk/credential-provider-node@3.535.0)(@pinecone-database/pinecone@2.1.0)(@supabase/supabase-js@2.43.4)(@xata.io/client@0.28.4(typescript@5.4.2))(axios@1.6.7)(cheerio@1.0.0-rc.12)(d3-dsv@2.0.0)(encoding@0.1.13)(epub2@3.0.2)(fast-xml-parser@4.3.5)(handlebars@4.7.8)(html-to-text@9.0.5)(ignore@5.2.4)(ioredis@5.3.2)(jsdom@23.0.1)(mammoth@1.7.2)(pdf-parse@1.1.1)(redis@4.6.13)(ws@8.14.2) ldapts: specifier: 4.2.6 version: 4.2.6 @@ -3993,6 +4005,10 @@ packages: '@mongodb-js/saslprep@1.1.0': resolution: {integrity: sha512-Xfijy7HvfzzqiOAhAepF4SGN5e9leLkMvg/OPOF97XemjfVCYN/oWa75wnkc6mltMSTwY+XlbhWgUOJmkFspSw==} + '@mozilla/readability@0.5.0': + resolution: {integrity: sha512-Z+CZ3QaosfFaTqvhQsIktyGrjFjSC0Fa4EMph4mqKnWhmyoGICsV/8QK+8HpXut6zV7zwfWwqDmEjtk1Qf6EgQ==} + engines: {node: '>=14.0.0'} + '@msgpackr-extract/msgpackr-extract-darwin-arm64@3.0.2': resolution: {integrity: sha512-9bfjwDxIDWmmOKusUcqdS4Rw+SETlp9Dy39Xui9BEGEk19dDwH0jhipwFzEff/pFg95NKymc6TOTbRKcWeRqyQ==} cpu: [arm64] @@ -6824,6 +6840,13 @@ packages: cheerio-select@1.6.0: resolution: {integrity: sha512-eq0GdBvxVFbqWgmCm7M3XGs1I8oLy/nExUnh6oLqmBditPO9AqQJrkslDpMun/hZ0yyTs8L0m85OHp4ho6Qm9g==} + cheerio-select@2.1.0: + resolution: {integrity: sha512-9v9kG0LvzrlcungtnJtpGNxY+fzECQKhK4EGJX2vByejiMX84MFNQw4UxPJl3bFbTMw+Dfs37XaIkCwTZfLh4g==} + + cheerio@1.0.0-rc.12: + resolution: {integrity: sha512-VqR8m68vM46BNnuZ5NtnGBKIE/DfN0cRIzg9n40EIq9NOv90ayxLBXA8fXC5gquFRGJSTRqBq25Jt2ECLR431Q==} + engines: {node: '>= 6'} + cheerio@1.0.0-rc.6: resolution: {integrity: sha512-hjx1XE1M/D5pAtMgvWwE21QClmAEeGHOIDfycgmndisdNgI6PE1cGRQkMGBcsbUbmEQyWu5PJLUcAOjtQS8DWw==} engines: {node: '>= 0.12'} @@ -10874,6 +10897,9 @@ packages: parse5-htmlparser2-tree-adapter@6.0.1: resolution: {integrity: sha512-qPuWvbLgvDGilKc5BoicRovlT4MtYT6JfJyBOMDsKoiT+GiuP5qyrPCnR9HcPECIJJmZh5jRndyNThnhhb/vlA==} + parse5-htmlparser2-tree-adapter@7.0.0: + resolution: {integrity: sha512-B77tOZrqqfUfnVcOrUvfdLbz4pu4RopLD/4vmu3HUPswwTA8OH0EMW9BlWR2B0RCoiZRAHEUu7IxeP1Pd1UU+g==} + parse5@6.0.1: resolution: {integrity: sha512-Ofn/CTFzRGTTxwpNEs9PP93gXShHcTq255nzRYSKe8AkVpZY7e1fpmTfOyoIvjP5HG7Z2ZM7VS9PPhQGW2pOpw==} @@ -16697,7 +16723,7 @@ snapshots: transitivePeerDependencies: - encoding - '@langchain/community@0.2.2(@aws-sdk/client-bedrock-runtime@3.535.0)(@aws-sdk/client-s3@3.478.0)(@aws-sdk/credential-provider-node@3.535.0)(@getzep/zep-js@0.9.0)(@google-ai/generativelanguage@2.5.0(encoding@0.1.13))(@google-cloud/storage@6.11.0(encoding@0.1.13))(@huggingface/inference@2.7.0)(@pinecone-database/pinecone@2.2.1)(@qdrant/js-client-rest@1.9.0(typescript@5.4.2))(@smithy/eventstream-codec@2.2.0)(@smithy/protocol-http@3.3.0)(@smithy/signature-v4@2.2.1)(@smithy/util-utf8@2.3.0)(@supabase/postgrest-js@1.15.2)(@supabase/supabase-js@2.43.4)(@xata.io/client@0.28.4(typescript@5.4.2))(axios@1.6.7)(cohere-ai@7.10.1(encoding@0.1.13))(d3-dsv@2.0.0)(encoding@0.1.13)(epub2@3.0.2(ts-toolbelt@9.6.0))(fast-xml-parser@4.3.5)(handlebars@4.7.8)(html-to-text@9.0.5)(ignore@5.2.4)(ioredis@5.3.2)(jsdom@23.0.1)(jsonwebtoken@9.0.2)(lodash@4.17.21)(mammoth@1.7.2)(mysql2@3.10.0)(pdf-parse@1.1.1)(pg@8.11.3)(redis@4.6.12)(ws@8.14.2)': + '@langchain/community@0.2.2(@aws-sdk/client-bedrock-runtime@3.535.0)(@aws-sdk/client-s3@3.478.0)(@aws-sdk/credential-provider-node@3.535.0)(@getzep/zep-js@0.9.0)(@google-ai/generativelanguage@2.5.0(encoding@0.1.13))(@google-cloud/storage@6.11.0(encoding@0.1.13))(@huggingface/inference@2.7.0)(@mozilla/readability@0.5.0)(@pinecone-database/pinecone@2.2.1)(@qdrant/js-client-rest@1.9.0(typescript@5.4.2))(@smithy/eventstream-codec@2.2.0)(@smithy/protocol-http@3.3.0)(@smithy/signature-v4@2.2.1)(@smithy/util-utf8@2.3.0)(@supabase/postgrest-js@1.15.2)(@supabase/supabase-js@2.43.4)(@xata.io/client@0.28.4(typescript@5.4.2))(axios@1.6.7)(cheerio@1.0.0-rc.12)(cohere-ai@7.10.1(encoding@0.1.13))(d3-dsv@2.0.0)(encoding@0.1.13)(epub2@3.0.2(ts-toolbelt@9.6.0))(fast-xml-parser@4.3.5)(handlebars@4.7.8)(html-to-text@9.0.5)(ignore@5.2.4)(ioredis@5.3.2)(jsdom@23.0.1)(jsonwebtoken@9.0.2)(lodash@4.17.21)(mammoth@1.7.2)(mysql2@3.10.0)(pdf-parse@1.1.1)(pg@8.11.3)(redis@4.6.12)(ws@8.14.2)': dependencies: '@langchain/core': 0.2.0 '@langchain/openai': 0.0.33(encoding@0.1.13) @@ -16705,7 +16731,7 @@ snapshots: expr-eval: 2.0.2 flat: 5.0.2 js-yaml: 4.1.0 - langchain: 0.2.2(@aws-sdk/client-s3@3.478.0)(@aws-sdk/credential-provider-node@3.535.0)(@google-ai/generativelanguage@2.5.0(encoding@0.1.13))(@pinecone-database/pinecone@2.2.1)(@supabase/supabase-js@2.43.4)(@xata.io/client@0.28.4(typescript@5.4.2))(axios@1.6.7)(d3-dsv@2.0.0)(encoding@0.1.13)(epub2@3.0.2(ts-toolbelt@9.6.0))(fast-xml-parser@4.3.5)(handlebars@4.7.8)(html-to-text@9.0.5)(ignore@5.2.4)(ioredis@5.3.2)(jsdom@23.0.1)(mammoth@1.7.2)(pdf-parse@1.1.1)(redis@4.6.12)(ws@8.14.2) + langchain: 0.2.2(@aws-sdk/client-s3@3.478.0)(@aws-sdk/credential-provider-node@3.535.0)(@google-ai/generativelanguage@2.5.0(encoding@0.1.13))(@pinecone-database/pinecone@2.2.1)(@supabase/supabase-js@2.43.4)(@xata.io/client@0.28.4(typescript@5.4.2))(axios@1.6.7)(cheerio@1.0.0-rc.12)(d3-dsv@2.0.0)(encoding@0.1.13)(epub2@3.0.2(ts-toolbelt@9.6.0))(fast-xml-parser@4.3.5)(handlebars@4.7.8)(html-to-text@9.0.5)(ignore@5.2.4)(ioredis@5.3.2)(jsdom@23.0.1)(mammoth@1.7.2)(pdf-parse@1.1.1)(redis@4.6.12)(ws@8.14.2) langsmith: 0.1.12 uuid: 9.0.1 zod: 3.23.8 @@ -16718,6 +16744,7 @@ snapshots: '@google-ai/generativelanguage': 2.5.0(encoding@0.1.13) '@google-cloud/storage': 6.11.0(encoding@0.1.13) '@huggingface/inference': 2.7.0 + '@mozilla/readability': 0.5.0 '@pinecone-database/pinecone': 2.2.1 '@qdrant/js-client-rest': 1.9.0(typescript@5.4.2) '@smithy/eventstream-codec': 2.2.0 @@ -16727,6 +16754,7 @@ snapshots: '@supabase/postgrest-js': 1.15.2 '@supabase/supabase-js': 2.43.4 '@xata.io/client': 0.28.4(typescript@5.4.2) + cheerio: 1.0.0-rc.12 cohere-ai: 7.10.1(encoding@0.1.13) d3-dsv: 2.0.0 epub2: 3.0.2(ts-toolbelt@9.6.0) @@ -16752,7 +16780,7 @@ snapshots: - pyodide - supports-color - '@langchain/community@0.2.2(@aws-sdk/client-bedrock-runtime@3.535.0)(@aws-sdk/client-s3@3.478.0)(@aws-sdk/credential-provider-node@3.535.0)(@getzep/zep-js@0.9.0)(@google-cloud/storage@6.11.0(encoding@0.1.13))(@huggingface/inference@2.7.0)(@pinecone-database/pinecone@2.1.0)(@qdrant/js-client-rest@1.9.0(typescript@5.4.2))(@smithy/eventstream-codec@2.2.0)(@smithy/protocol-http@3.3.0)(@smithy/signature-v4@2.2.1)(@smithy/util-utf8@2.3.0)(@supabase/postgrest-js@1.15.2)(@supabase/supabase-js@2.43.4)(@xata.io/client@0.28.4(typescript@5.4.2))(axios@1.6.7)(cohere-ai@7.10.1(encoding@0.1.13))(d3-dsv@2.0.0)(encoding@0.1.13)(epub2@3.0.2)(fast-xml-parser@4.3.5)(handlebars@4.7.8)(html-to-text@9.0.5)(ignore@5.2.4)(ioredis@5.3.2)(jsdom@23.0.1)(jsonwebtoken@9.0.2)(lodash@4.17.21)(mammoth@1.7.2)(mysql2@3.10.0)(pdf-parse@1.1.1)(pg@8.11.3)(redis@4.6.13)(ws@8.14.2)': + '@langchain/community@0.2.2(@aws-sdk/client-bedrock-runtime@3.535.0)(@aws-sdk/client-s3@3.478.0)(@aws-sdk/credential-provider-node@3.535.0)(@getzep/zep-js@0.9.0)(@google-cloud/storage@6.11.0(encoding@0.1.13))(@huggingface/inference@2.7.0)(@mozilla/readability@0.5.0)(@pinecone-database/pinecone@2.1.0)(@qdrant/js-client-rest@1.9.0(typescript@5.4.2))(@smithy/eventstream-codec@2.2.0)(@smithy/protocol-http@3.3.0)(@smithy/signature-v4@2.2.1)(@smithy/util-utf8@2.3.0)(@supabase/postgrest-js@1.15.2)(@supabase/supabase-js@2.43.4)(@xata.io/client@0.28.4(typescript@5.4.2))(axios@1.6.7)(cheerio@1.0.0-rc.12)(cohere-ai@7.10.1(encoding@0.1.13))(d3-dsv@2.0.0)(encoding@0.1.13)(epub2@3.0.2)(fast-xml-parser@4.3.5)(handlebars@4.7.8)(html-to-text@9.0.5)(ignore@5.2.4)(ioredis@5.3.2)(jsdom@23.0.1)(jsonwebtoken@9.0.2)(lodash@4.17.21)(mammoth@1.7.2)(mysql2@3.10.0)(pdf-parse@1.1.1)(pg@8.11.3)(redis@4.6.13)(ws@8.14.2)': dependencies: '@langchain/core': 0.2.0 '@langchain/openai': 0.0.33(encoding@0.1.13) @@ -16760,7 +16788,7 @@ snapshots: expr-eval: 2.0.2 flat: 5.0.2 js-yaml: 4.1.0 - langchain: 0.2.2(@aws-sdk/client-s3@3.478.0)(@aws-sdk/credential-provider-node@3.535.0)(@pinecone-database/pinecone@2.1.0)(@supabase/supabase-js@2.43.4)(@xata.io/client@0.28.4(typescript@5.4.2))(axios@1.6.7)(d3-dsv@2.0.0)(encoding@0.1.13)(epub2@3.0.2)(fast-xml-parser@4.3.5)(handlebars@4.7.8)(html-to-text@9.0.5)(ignore@5.2.4)(ioredis@5.3.2)(jsdom@23.0.1)(mammoth@1.7.2)(pdf-parse@1.1.1)(redis@4.6.13)(ws@8.14.2) + langchain: 0.2.2(@aws-sdk/client-s3@3.478.0)(@aws-sdk/credential-provider-node@3.535.0)(@pinecone-database/pinecone@2.1.0)(@supabase/supabase-js@2.43.4)(@xata.io/client@0.28.4(typescript@5.4.2))(axios@1.6.7)(cheerio@1.0.0-rc.12)(d3-dsv@2.0.0)(encoding@0.1.13)(epub2@3.0.2)(fast-xml-parser@4.3.5)(handlebars@4.7.8)(html-to-text@9.0.5)(ignore@5.2.4)(ioredis@5.3.2)(jsdom@23.0.1)(mammoth@1.7.2)(pdf-parse@1.1.1)(redis@4.6.13)(ws@8.14.2) langsmith: 0.1.12 uuid: 9.0.1 zod: 3.23.8 @@ -16772,6 +16800,7 @@ snapshots: '@getzep/zep-js': 0.9.0 '@google-cloud/storage': 6.11.0(encoding@0.1.13) '@huggingface/inference': 2.7.0 + '@mozilla/readability': 0.5.0 '@pinecone-database/pinecone': 2.1.0 '@qdrant/js-client-rest': 1.9.0(typescript@5.4.2) '@smithy/eventstream-codec': 2.2.0 @@ -16781,6 +16810,7 @@ snapshots: '@supabase/postgrest-js': 1.15.2 '@supabase/supabase-js': 2.43.4 '@xata.io/client': 0.28.4(typescript@5.4.2) + cheerio: 1.0.0-rc.12 cohere-ai: 7.10.1(encoding@0.1.13) d3-dsv: 2.0.0 epub2: 3.0.2(ts-toolbelt@9.6.0) @@ -16973,6 +17003,8 @@ snapshots: dependencies: sparse-bitfield: 3.0.3 + '@mozilla/readability@0.5.0': {} + '@msgpackr-extract/msgpackr-extract-darwin-arm64@3.0.2': optional: true @@ -20906,6 +20938,25 @@ snapshots: domhandler: 4.3.1 domutils: 2.8.0 + cheerio-select@2.1.0: + dependencies: + boolbase: 1.0.0 + css-select: 5.1.0 + css-what: 6.1.0 + domelementtype: 2.3.0 + domhandler: 5.0.3 + domutils: 3.0.1 + + cheerio@1.0.0-rc.12: + dependencies: + cheerio-select: 2.1.0 + dom-serializer: 2.0.0 + domhandler: 5.0.3 + domutils: 3.0.1 + htmlparser2: 8.0.2 + parse5: 7.1.2 + parse5-htmlparser2-tree-adapter: 7.0.0 + cheerio@1.0.0-rc.6: dependencies: cheerio-select: 1.6.0 @@ -24423,7 +24474,7 @@ snapshots: kuler@2.0.0: {} - langchain@0.2.2(@aws-sdk/client-s3@3.478.0)(@aws-sdk/credential-provider-node@3.535.0)(@google-ai/generativelanguage@2.5.0(encoding@0.1.13))(@pinecone-database/pinecone@2.2.1)(@supabase/supabase-js@2.43.4)(@xata.io/client@0.28.4(typescript@5.4.2))(axios@1.6.7)(d3-dsv@2.0.0)(encoding@0.1.13)(epub2@3.0.2(ts-toolbelt@9.6.0))(fast-xml-parser@4.3.5)(handlebars@4.7.8)(html-to-text@9.0.5)(ignore@5.2.4)(ioredis@5.3.2)(jsdom@23.0.1)(mammoth@1.7.2)(pdf-parse@1.1.1)(redis@4.6.12)(ws@8.14.2): + langchain@0.2.2(@aws-sdk/client-s3@3.478.0)(@aws-sdk/credential-provider-node@3.535.0)(@google-ai/generativelanguage@2.5.0(encoding@0.1.13))(@pinecone-database/pinecone@2.2.1)(@supabase/supabase-js@2.43.4)(@xata.io/client@0.28.4(typescript@5.4.2))(axios@1.6.7)(cheerio@1.0.0-rc.12)(d3-dsv@2.0.0)(encoding@0.1.13)(epub2@3.0.2(ts-toolbelt@9.6.0))(fast-xml-parser@4.3.5)(handlebars@4.7.8)(html-to-text@9.0.5)(ignore@5.2.4)(ioredis@5.3.2)(jsdom@23.0.1)(mammoth@1.7.2)(pdf-parse@1.1.1)(redis@4.6.12)(ws@8.14.2): dependencies: '@langchain/core': 0.2.0 '@langchain/openai': 0.0.33(encoding@0.1.13) @@ -24449,6 +24500,7 @@ snapshots: '@supabase/supabase-js': 2.43.4 '@xata.io/client': 0.28.4(typescript@5.4.2) axios: 1.6.7(debug@3.2.7) + cheerio: 1.0.0-rc.12 d3-dsv: 2.0.0 epub2: 3.0.2(ts-toolbelt@9.6.0) fast-xml-parser: 4.3.5 @@ -24465,7 +24517,7 @@ snapshots: - encoding - supports-color - langchain@0.2.2(@aws-sdk/client-s3@3.478.0)(@aws-sdk/credential-provider-node@3.535.0)(@pinecone-database/pinecone@2.1.0)(@supabase/supabase-js@2.43.4)(@xata.io/client@0.28.4(typescript@5.4.2))(axios@1.6.7)(d3-dsv@2.0.0)(encoding@0.1.13)(epub2@3.0.2)(fast-xml-parser@4.3.5)(handlebars@4.7.8)(html-to-text@9.0.5)(ignore@5.2.4)(ioredis@5.3.2)(jsdom@23.0.1)(mammoth@1.7.2)(pdf-parse@1.1.1)(redis@4.6.13)(ws@8.14.2): + langchain@0.2.2(@aws-sdk/client-s3@3.478.0)(@aws-sdk/credential-provider-node@3.535.0)(@pinecone-database/pinecone@2.1.0)(@supabase/supabase-js@2.43.4)(@xata.io/client@0.28.4(typescript@5.4.2))(axios@1.6.7)(cheerio@1.0.0-rc.12)(d3-dsv@2.0.0)(encoding@0.1.13)(epub2@3.0.2)(fast-xml-parser@4.3.5)(handlebars@4.7.8)(html-to-text@9.0.5)(ignore@5.2.4)(ioredis@5.3.2)(jsdom@23.0.1)(mammoth@1.7.2)(pdf-parse@1.1.1)(redis@4.6.13)(ws@8.14.2): dependencies: '@langchain/core': 0.2.0 '@langchain/openai': 0.0.33(encoding@0.1.13) @@ -24490,6 +24542,7 @@ snapshots: '@supabase/supabase-js': 2.43.4 '@xata.io/client': 0.28.4(typescript@5.4.2) axios: 1.6.7(debug@3.2.7) + cheerio: 1.0.0-rc.12 d3-dsv: 2.0.0 epub2: 3.0.2(ts-toolbelt@9.6.0) fast-xml-parser: 4.3.5 @@ -25708,6 +25761,11 @@ snapshots: dependencies: parse5: 6.0.1 + parse5-htmlparser2-tree-adapter@7.0.0: + dependencies: + domhandler: 5.0.3 + parse5: 7.1.2 + parse5@6.0.1: {} parse5@7.1.2: