From 8d4974996e8ec718f0b70e5d81ea8bb909730e7c Mon Sep 17 00:00:00 2001 From: Jorge Cortes Date: Mon, 6 May 2024 12:11:49 -0500 Subject: [PATCH] Migrated to latest rest api version (#11707) --- .../actions/delete-vectors/delete-vectors.mjs | 37 ++-- .../actions/fetch-vectors/fetch-vectors.mjs | 34 ++-- .../pinecone/actions/query-ids/query-ids.mjs | 39 ++-- .../actions/update-vector/update-vector.mjs | 29 ++- .../actions/upsert-vector/upsert-vector.mjs | 27 +-- components/pinecone/common/constants.mjs | 26 +-- components/pinecone/package.json | 2 +- components/pinecone/pinecone.app.mjs | 166 +++--------------- 8 files changed, 100 insertions(+), 260 deletions(-) diff --git a/components/pinecone/actions/delete-vectors/delete-vectors.mjs b/components/pinecone/actions/delete-vectors/delete-vectors.mjs index 9855c6e3b85b3..0e38590a69832 100644 --- a/components/pinecone/actions/delete-vectors/delete-vectors.mjs +++ b/components/pinecone/actions/delete-vectors/delete-vectors.mjs @@ -4,9 +4,9 @@ import utils from "../../common/utils.mjs"; export default { key: "pinecone-delete-vectors", name: "Delete Vectors", - description: "Deletes one or more vectors by ID, from a single namespace. [See the docs](https://docs.pinecone.io/reference/delete_post).", + description: "Deletes one or more vectors by ID, from a single namespace. [See the documentation](https://docs.pinecone.io/reference/api/data-plane/delete).", type: "action", - version: "0.0.1", + version: "0.0.2", props: { app, indexName: { @@ -15,10 +15,16 @@ export default { "indexName", ], }, - projectId: { + prefix: { propDefinition: [ app, - "projectId", + "prefix", + ], + }, + namespace: { + propDefinition: [ + app, + "namespace", ], }, ids: { @@ -28,24 +34,12 @@ export default { propDefinition: [ app, "vectorId", - ({ - indexName, projectId, - }) => ({ - indexName, - projectId, - }), - ], - }, - namespace: { - propDefinition: [ - app, - "namespace", ], }, }, methods: { deleteVector(args = {}) { - return this.app.create({ + return this.app.post({ path: "/vectors/delete", ...args, }); @@ -53,14 +47,14 @@ export default { }, async run({ $: step }) { const { + deleteVector, indexName, - projectId, ids, namespace, } = this; - await this.deleteVector({ - projectId, + await deleteVector({ + step, indexName, data: { ids: utils.parseArray(ids), @@ -69,5 +63,8 @@ export default { }); step.export("$summary", "Successfully deleted vectors"); + return { + success: true, + }; }, }; diff --git a/components/pinecone/actions/fetch-vectors/fetch-vectors.mjs b/components/pinecone/actions/fetch-vectors/fetch-vectors.mjs index 54d1a9b50f3c7..bdd62171b2c79 100644 --- a/components/pinecone/actions/fetch-vectors/fetch-vectors.mjs +++ b/components/pinecone/actions/fetch-vectors/fetch-vectors.mjs @@ -4,9 +4,9 @@ import app from "../../pinecone.app.mjs"; export default { key: "pinecone-fetch-vectors", name: "Fetch Vectors", - description: "Looks up and returns vectors by ID, from a single namespace.. [See the docs](https://docs.pinecone.io/reference/fetch).", + description: "Looks up and returns vectors by ID, from a single namespace.. [See the documentation](https://docs.pinecone.io/reference/api/data-plane/fetch).", type: "action", - version: "0.0.1", + version: "0.0.2", props: { app, indexName: { @@ -15,10 +15,10 @@ export default { "indexName", ], }, - projectId: { + namespace: { propDefinition: [ app, - "projectId", + "namespace", ], }, ids: { @@ -28,31 +28,27 @@ export default { propDefinition: [ app, "vectorId", - ({ - indexName, projectId, - }) => ({ - indexName, - projectId, - }), ], }, - namespace: { - propDefinition: [ - app, - "namespace", - ], + }, + methods: { + fetchVectors(args = {}) { + return this.app.makeRequest({ + path: "/vectors/fetch", + ...args, + }); }, }, async run({ $: step }) { const { + fetchVectors, indexName, - projectId, ids, namespace, } = this; - const response = await this.app.fetchVectors({ - projectId, + const response = await fetchVectors({ + step, indexName, params: { ids: utils.parseArray(ids), @@ -60,7 +56,7 @@ export default { }, }); - step.export("$summary", `Successfully fetched ${Object.keys(response?.vectors).length} vectors.`); + step.export("$summary", `Successfully fetched \`${Object.keys(response.vectors).length}\` vector(s).`); return response; }, diff --git a/components/pinecone/actions/query-ids/query-ids.mjs b/components/pinecone/actions/query-ids/query-ids.mjs index 5f99a0a86e449..ab4267a8a658c 100644 --- a/components/pinecone/actions/query-ids/query-ids.mjs +++ b/components/pinecone/actions/query-ids/query-ids.mjs @@ -4,9 +4,9 @@ import utils from "../../common/utils.mjs"; export default { key: "pinecone-query-ids", name: "Query IDs", - description: "Searches a namespace, using a query vector. It retrieves the ids of the most similar items in a namespace, along with their similarity scores. [See the docs](https://docs.pinecone.io/reference/query).", + description: "Searches a namespace, using a query vector. It retrieves the ids of the most similar items in a namespace, along with their similarity scores. [See the documentation](https://docs.pinecone.io/reference/api/data-plane/query).", type: "action", - version: "0.0.1", + version: "0.0.2", props: { app, indexName: { @@ -15,12 +15,6 @@ export default { "indexName", ], }, - projectId: { - propDefinition: [ - app, - "projectId", - ], - }, topK: { type: "integer", label: "Top K", @@ -32,7 +26,7 @@ export default { filter: { type: "object", label: "Filter", - description: "The filter to apply. You can use vector metadata to limit your search. [See the docs](https://www.pinecone.io/docs/metadata-filtering/).", + description: "The filter to apply. You can use vector metadata to limit your search. For guidance and examples, see [filtering-with-metadata](https://docs.pinecone.io/guides/data/filtering-with-metadata).", optional: true, }, includeValues: { @@ -59,16 +53,9 @@ export default { propDefinition: [ app, "vectorId", - ({ - indexName, projectId, - }) => ({ - indexName, - projectId, - }), ], }, vector: { - optional: true, description: `${app.propDefinitions.vectorValues.description} Each request can contain only one of the parameters either **Vector Values** or **Vector ID**.`, propDefinition: [ app, @@ -76,10 +63,18 @@ export default { ], }, }, + methods: { + query(args = {}) { + return this.app.post({ + path: "/query", + ...args, + }); + }, + }, async run({ $: step }) { const { + query, indexName, - projectId, id, vector, topK, @@ -89,12 +84,16 @@ export default { namespace, } = this; - const response = await this.app.query({ - projectId, + const vectorParsed = utils.parseArray(vector); + + const response = await query({ + step, indexName, data: { id, - vector: utils.parseArray(vector), + ...(vectorParsed.length && { + vector: vectorParsed, + }), topK, filter: utils.parse(filter), includeValues, diff --git a/components/pinecone/actions/update-vector/update-vector.mjs b/components/pinecone/actions/update-vector/update-vector.mjs index 688c6b332819e..f8ed46d514702 100644 --- a/components/pinecone/actions/update-vector/update-vector.mjs +++ b/components/pinecone/actions/update-vector/update-vector.mjs @@ -4,9 +4,9 @@ import app from "../../pinecone.app.mjs"; export default { key: "pinecone-update-vector", name: "Update Vector", - description: "Updates vector in a namespace. If a value is included, it will overwrite the previous value. [See the docs](https://docs.pinecone.io/reference/update).", + description: "Updates vector in a namespace. If a value is included, it will overwrite the previous value. [See the documentation](https://docs.pinecone.io/reference/api/data-plane/update).", type: "action", - version: "0.0.1", + version: "0.0.2", props: { app, indexName: { @@ -15,22 +15,10 @@ export default { "indexName", ], }, - projectId: { - propDefinition: [ - app, - "projectId", - ], - }, id: { propDefinition: [ app, "vectorId", - ({ - indexName, projectId, - }) => ({ - indexName, - projectId, - }), ], }, values: { @@ -54,7 +42,7 @@ export default { }, methods: { updateVector(args = {}) { - return this.app.create({ + return this.app.post({ path: "/vectors/update", ...args, }); @@ -62,25 +50,28 @@ export default { }, async run({ $: step }) { const { + updateVector, indexName, - projectId, id, values, metadata, namespace, } = this; - await this.updateVector({ - projectId, + await updateVector({ + step, indexName, data: { id, - values, + values: utils.parseArray(values), setMetadata: utils.parse(metadata), namespace, }, }); step.export("$summary", `Successfully updated vector with ID ${id}.`); + return { + success: true, + }; }, }; diff --git a/components/pinecone/actions/upsert-vector/upsert-vector.mjs b/components/pinecone/actions/upsert-vector/upsert-vector.mjs index 8e8a8e883f1fb..48663bb8b1afc 100644 --- a/components/pinecone/actions/upsert-vector/upsert-vector.mjs +++ b/components/pinecone/actions/upsert-vector/upsert-vector.mjs @@ -4,9 +4,9 @@ import utils from "../../common/utils.mjs"; export default { key: "pinecone-upsert-vector", name: "Upsert Vector", - description: "Writes vectors into a namespace. If a new value is upserted for an existing vector ID, it will overwrite the previous value. [See the docs](https://docs.pinecone.io/reference/upsert).", + description: "Writes vectors into a namespace. If a new value is upserted for an existing vector ID, it will overwrite the previous value. [See the documentation](https://docs.pinecone.io/reference/api/data-plane/upsert).", type: "action", - version: "0.0.1", + version: "0.0.2", props: { app, indexName: { @@ -15,25 +15,14 @@ export default { "indexName", ], }, - projectId: { - propDefinition: [ - app, - "projectId", - ], - }, id: { propDefinition: [ app, "vectorId", - ({ - indexName, projectId, - }) => ({ - indexName, - projectId, - }), ], }, values: { + optional: false, propDefinition: [ app, "vectorValues", @@ -54,7 +43,7 @@ export default { }, methods: { upsertVector(args = {}) { - return this.app.create({ + return this.app.post({ path: "/vectors/upsert", ...args, }); @@ -62,22 +51,22 @@ export default { }, async run({ $: step }) { const { + upsertVector, indexName, - projectId, id, values, metadata, namespace, } = this; - const response = await this.upsertVector({ - projectId, + const response = await upsertVector({ + step, indexName, data: { vectors: [ { id, - values, + values: utils.parseArray(values), metadata: utils.parse(metadata), }, ], diff --git a/components/pinecone/common/constants.mjs b/components/pinecone/common/constants.mjs index 3ddf0a85a967a..25613cf3f06ce 100644 --- a/components/pinecone/common/constants.mjs +++ b/components/pinecone/common/constants.mjs @@ -1,23 +1,8 @@ const INDEX_NAME_PLACEHOLDER = ""; -const PROJECT_ID_PLACEHOLDER = ""; -const ENV_PLACEHOLDER = ""; -const BASE_URL = `https://${INDEX_NAME_PLACEHOLDER}-${PROJECT_ID_PLACEHOLDER}.svc.${ENV_PLACEHOLDER}.pinecone.io`; -const CONTROLLER_URL = `https://controller.${ENV_PLACEHOLDER}.pinecone.io`; -const LAST_CREATED_AT = "lastCreatedAt"; +const BASE_URL = `https://${INDEX_NAME_PLACEHOLDER}`; +const CONTROLLER_URL = "https://api.pinecone.io"; const DEFAULT_MAX = 600; -const FILE_PROP_NAMES = [ - "attachment", - "uploaddoc", - "upload_file", -]; - -const CONTENT_TYPE_KEY_HEADER = "Content-Type"; -const MULTIPART_FORM_DATA_VALUE_HEADER = "multipart/form-data"; -const MULTIPART_FORM_DATA_HEADERS = { - [CONTENT_TYPE_KEY_HEADER]: MULTIPART_FORM_DATA_VALUE_HEADER, -}; - const API = { DEFAULT: "default", CONTROLLER: "controller", @@ -25,15 +10,8 @@ const API = { export default { API, - ENV_PLACEHOLDER, - PROJECT_ID_PLACEHOLDER, INDEX_NAME_PLACEHOLDER, BASE_URL, CONTROLLER_URL, DEFAULT_MAX, - LAST_CREATED_AT, - FILE_PROP_NAMES, - CONTENT_TYPE_KEY_HEADER, - MULTIPART_FORM_DATA_VALUE_HEADER, - MULTIPART_FORM_DATA_HEADERS, }; diff --git a/components/pinecone/package.json b/components/pinecone/package.json index 2698522ba74ae..a1141179b4f8f 100644 --- a/components/pinecone/package.json +++ b/components/pinecone/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/pinecone", - "version": "0.0.3", + "version": "0.0.4", "description": "Pipedream Pinecone Components", "main": "pinecone.app.mjs", "keywords": [ diff --git a/components/pinecone/pinecone.app.mjs b/components/pinecone/pinecone.app.mjs index 292adf8610a69..a74e8c2e75e6a 100644 --- a/components/pinecone/pinecone.app.mjs +++ b/components/pinecone/pinecone.app.mjs @@ -9,44 +9,22 @@ export default { indexName: { type: "string", label: "Index Name", - description: "The name of the index. E.g. `my-index`", - options() { - return this.listIndexes(); + description: "The name of the index. E.g. `my-index.pinecone.io`", + async options() { + const { indexes } = await this.listIndexes(); + return indexes.map(({ + name: label, host: value, + }) => ({ + label, + value, + })); }, }, - projectId: { + prefix: { type: "string", - label: "Project ID", - description: "The ID of the project. You can get the project from the Pinecone URL, for example https://app.pinecone.io/organizations/abcde/projects/us-east-1-aws:611e7f9/indexes, then the Project ID is `611e7f9`", - }, - vectorId: { - type: "string", - label: "Vector ID", - description: "The ID of the vector.", - async options({ - indexName, projectId, - }) { - if (!indexName || !projectId) { - return []; - } - - const { database: { dimension } } = await this.describeIndex({ - indexName, - }); - - const { matches } = await this.query({ - projectId, - indexName, - data: { - topK: dimension, - vector: Array.from({ - length: dimension, - }).map(() => 0), - }, - }); - - return matches?.map(({ id }) => id); - }, + label: "Prefix", + description: "The vector IDs to fetch. Does not accept values containing spaces. E.g. `my-prefix`", + optional: true, }, namespace: { type: "string", @@ -54,10 +32,16 @@ export default { description: "The namespace to use.", optional: true, }, + vectorId: { + type: "string", + label: "Vector ID", + description: "The ID of the vector.", + }, vectorValues: { type: "string[]", label: "Vector Values", description: "The values of the vector.", + optional: true, }, vectorMetadata: { type: "object", @@ -68,25 +52,17 @@ export default { }, methods: { getBaseUrl({ - api, projectId, indexName, + api, indexName, } = {}) { - const { environment } = this.$auth; return api === constants.API.CONTROLLER ? constants.CONTROLLER_URL - .replace(constants.ENV_PLACEHOLDER, environment) : constants.BASE_URL - .replace(constants.INDEX_NAME_PLACEHOLDER, indexName) - .replace(constants.PROJECT_ID_PLACEHOLDER, projectId) - .replace(constants.ENV_PLACEHOLDER, environment); + .replace(constants.INDEX_NAME_PLACEHOLDER, indexName); }, getUrl({ - api, path, projectId, indexName, + path, ...args } = {}) { - const baseUrl = this.getBaseUrl({ - api, - projectId, - indexName, - }); + const baseUrl = this.getBaseUrl(args); return `${baseUrl}${path}`; }, getHeaders(headers) { @@ -102,118 +78,32 @@ export default { }); }, makeRequest({ - step = this, api, headers, projectId, indexName, path, ...args + step = this, api, headers, indexName, path, ...args } = {}) { - const config = { + ...args, paramsSerializer: this.paramsSerializer, headers: this.getHeaders(headers), url: this.getUrl({ api, indexName, - projectId, path, }), - ...args, }; - return axios(step, config); }, - create(args = {}) { + post(args = {}) { return this.makeRequest({ method: "post", ...args, }); }, - update(args = {}) { - return this.makeRequest({ - method: "put", - ...args, - }); - }, - delete(args = {}) { - return this.makeRequest({ - method: "delete", - ...args, - }); - }, - patch(args = {}) { - return this.makeRequest({ - method: "patch", - ...args, - }); - }, - query(args = {}) { - return this.create({ - path: "/query", - ...args, - }); - }, - describeIndex({ - indexName, ...args - } = {}) { + listIndexes(args = {}) { return this.makeRequest({ api: constants.API.CONTROLLER, - path: `/databases/${indexName}`, + path: "/indexes", ...args, }); }, - listIndexes() { - return this.makeRequest({ - api: constants.API.CONTROLLER, - path: "/databases", - }); - }, - fetchVectors(args = {}) { - return this.makeRequest({ - path: "/vectors/fetch", - ...args, - }); - }, - async *getResourcesStream({ - resourceFn, - resourceFnArgs, - resourceName, - lastCreatedAt, - max = constants.DEFAULT_MAX, - }) { - let page = 1; - let resourcesCount = 0; - - while (true) { - const response = - await resourceFn({ - ...resourceFnArgs, - params: { - ...resourceFnArgs.params, - page, - }, - }); - - const nextResources = resourceName && response[resourceName] || response; - - if (!nextResources?.length) { - console.log("No more resources found"); - return; - } - - for (const resource of nextResources) { - const dateFilter = - lastCreatedAt - && Date.parse(resource.created_at) > Date.parse(lastCreatedAt); - - if (!lastCreatedAt || dateFilter) { - yield resource; - resourcesCount += 1; - } - - if (resourcesCount >= max) { - return; - } - } - - page += 1; - } - }, }, };