From afc6932430e716b361fdd12e03b8d78b6bf2639e Mon Sep 17 00:00:00 2001 From: dom_ Date: Tue, 14 Nov 2023 12:26:52 -0500 Subject: [PATCH 1/9] Add optional custom payload param and 1 test --- langchain/src/vectorstores/qdrant.ts | 29 ++++++++--- .../src/vectorstores/tests/qdrant.test.ts | 50 +++++++++++++++++++ 2 files changed, 72 insertions(+), 7 deletions(-) diff --git a/langchain/src/vectorstores/qdrant.ts b/langchain/src/vectorstores/qdrant.ts index 6ec54233a6ec..43e524c0a3b8 100644 --- a/langchain/src/vectorstores/qdrant.ts +++ b/langchain/src/vectorstores/qdrant.ts @@ -84,13 +84,18 @@ export class QdrantVectorStore extends VectorStore { * from the documents using the `Embeddings` instance and then adds the * vectors to the database. * @param documents Array of `Document` instances to be added to the Qdrant database. + * @param customPayload Optional 'object' in JSON format used to query database on more criteria * @returns Promise that resolves when the documents have been added to the database. */ - async addDocuments(documents: Document[]): Promise { + async addDocuments( + documents: Document[], + customPayload?: object + ): Promise { const texts = documents.map(({ pageContent }) => pageContent); await this.addVectors( await this.embeddings.embedDocuments(texts), - documents + documents, + customPayload ); } @@ -100,9 +105,14 @@ export class QdrantVectorStore extends VectorStore { * database. * @param vectors Array of vectors to be added to the Qdrant database. * @param documents Array of `Document` instances associated with the vectors. + * @param customPayload Optional 'object' in JSON format used to query database on more criteria * @returns Promise that resolves when the vectors have been added to the database. */ - async addVectors(vectors: number[][], documents: Document[]): Promise { + async addVectors( + vectors: number[][], + documents: Document[], + customPayload?: object + ): Promise { if (vectors.length === 0) { return; } @@ -115,6 +125,7 @@ export class QdrantVectorStore extends VectorStore { payload: { content: documents[idx].pageContent, metadata: documents[idx].metadata, + ...customPayload, }, })); @@ -204,13 +215,15 @@ export class QdrantVectorStore extends VectorStore { * @param metadatas Array or single object of metadata to be associated with the texts. * @param embeddings `Embeddings` instance used to generate vectors from the texts. * @param dbConfig `QdrantLibArgs` instance specifying the configuration for the Qdrant database. + * @param customPayload Optional 'object' in JSON format used to query database on more criteria * @returns Promise that resolves with a new `QdrantVectorStore` instance. */ static async fromTexts( texts: string[], metadatas: object[] | object, embeddings: Embeddings, - dbConfig: QdrantLibArgs + dbConfig: QdrantLibArgs, + customPayload? : object ): Promise { const docs = []; for (let i = 0; i < texts.length; i += 1) { @@ -221,7 +234,7 @@ export class QdrantVectorStore extends VectorStore { }); docs.push(newDoc); } - return QdrantVectorStore.fromDocuments(docs, embeddings, dbConfig); + return QdrantVectorStore.fromDocuments(docs, embeddings, dbConfig, customPayload); } /** @@ -230,15 +243,17 @@ export class QdrantVectorStore extends VectorStore { * @param docs Array of `Document` instances to be added to the Qdrant database. * @param embeddings `Embeddings` instance used to generate vectors from the documents. * @param dbConfig `QdrantLibArgs` instance specifying the configuration for the Qdrant database. + * @param customPayload Optional 'object' in JSON format used to query database on more criteria * @returns Promise that resolves with a new `QdrantVectorStore` instance. */ static async fromDocuments( docs: Document[], embeddings: Embeddings, - dbConfig: QdrantLibArgs + dbConfig: QdrantLibArgs, + customPayload?: object ): Promise { const instance = new this(embeddings, dbConfig); - await instance.addDocuments(docs); + await instance.addDocuments(docs, customPayload); return instance; } diff --git a/langchain/src/vectorstores/tests/qdrant.test.ts b/langchain/src/vectorstores/tests/qdrant.test.ts index 38d5a8c1c85e..29ea24e3565a 100644 --- a/langchain/src/vectorstores/tests/qdrant.test.ts +++ b/langchain/src/vectorstores/tests/qdrant.test.ts @@ -31,3 +31,53 @@ test("QdrantVectorStore works", async () => { expect(results).toHaveLength(0); }); + +test("QdrantVectorStore adds vectors with custom payload", async () => { + // Mock Qdrant client + const client = { + upsert: jest.fn(), + search: jest.fn().mockResolvedValue([]), + getCollections: jest.fn().mockResolvedValue({ collections: [] }), + createCollection: jest.fn(), + }; + + // Mock embeddings + const embeddings = new FakeEmbeddings(); + + // Create QdrantVectorStore instance with the mock client + const qdrantVectorStore = new QdrantVectorStore(embeddings, { + client: client as any, + }); + + // Define a custom payload + const customPayload = { + customField1: "value1", + customField2: "value2", + }; + + // Add documents with custom payload + await qdrantVectorStore.addDocuments( + [ + { + pageContent: "hello", + metadata: {}, + }, + ], + customPayload + ); + + // Verify that the Qdrant client's upsert method was called with the correct arguments + expect(client.upsert).toHaveBeenCalledTimes(1); + expect(client.upsert).toHaveBeenCalledWith("documents", { + wait: true, + points: [ + expect.objectContaining({ + payload: expect.objectContaining({ + content: "hello", + metadata: {}, + ...customPayload, + }), + }), + ], + }); +}); From f28b15b749ed940e2a887ae12625cba76988d699 Mon Sep 17 00:00:00 2001 From: dom_ Date: Tue, 21 Nov 2023 13:14:22 -0500 Subject: [PATCH 2/9] Add Custom Payload in Documents --- langchain/src/document.ts | 3 ++ langchain/src/vectorstores/qdrant.ts | 6 ++- .../src/vectorstores/tests/qdrant.test.ts | 50 +++++++++++++++++++ 3 files changed, 57 insertions(+), 2 deletions(-) diff --git a/langchain/src/document.ts b/langchain/src/document.ts index 3647d51609ff..b2ef023d6bdc 100644 --- a/langchain/src/document.ts +++ b/langchain/src/document.ts @@ -5,6 +5,7 @@ export interface DocumentInput< pageContent: string; metadata?: Metadata; + customPayload?: object; } /** @@ -19,6 +20,8 @@ export class Document< metadata: Metadata; + customPayload?: object | undefined; + constructor(fields: DocumentInput) { this.pageContent = fields.pageContent ? fields.pageContent.toString() diff --git a/langchain/src/vectorstores/qdrant.ts b/langchain/src/vectorstores/qdrant.ts index 43e524c0a3b8..9ff550cd49c2 100644 --- a/langchain/src/vectorstores/qdrant.ts +++ b/langchain/src/vectorstores/qdrant.ts @@ -125,7 +125,8 @@ export class QdrantVectorStore extends VectorStore { payload: { content: documents[idx].pageContent, metadata: documents[idx].metadata, - ...customPayload, + ...(documents[idx].customPayload ? documents[idx].customPayload != customPayload ? + documents[idx].customPayload : customPayload : customPayload), }, })); @@ -231,10 +232,11 @@ export class QdrantVectorStore extends VectorStore { const newDoc = new Document({ pageContent: texts[i], metadata, + customPayload: customPayload, }); docs.push(newDoc); } - return QdrantVectorStore.fromDocuments(docs, embeddings, dbConfig, customPayload); + return QdrantVectorStore.fromDocuments(docs, embeddings, dbConfig); } /** diff --git a/langchain/src/vectorstores/tests/qdrant.test.ts b/langchain/src/vectorstores/tests/qdrant.test.ts index 29ea24e3565a..edc9f8c24b62 100644 --- a/langchain/src/vectorstores/tests/qdrant.test.ts +++ b/langchain/src/vectorstores/tests/qdrant.test.ts @@ -81,3 +81,53 @@ test("QdrantVectorStore adds vectors with custom payload", async () => { ], }); }); + +test("QdrantVectorStore adds vectors with custom payload in Document", async () => { + // Mock Qdrant client + const client = { + upsert: jest.fn(), + search: jest.fn().mockResolvedValue([]), + getCollections: jest.fn().mockResolvedValue({ collections: [] }), + createCollection: jest.fn(), + }; + + // Mock embeddings + const embeddings = new FakeEmbeddings(); + + // Create QdrantVectorStore instance with the mock client + const qdrantVectorStore = new QdrantVectorStore(embeddings, { + client: client as any, + }); + + // Define a custom payload + const customPayload = { + customField1: "value1", + customField2: "value2", + }; + + // Add documents with custom payload + await qdrantVectorStore.addDocuments( + [ + { + pageContent: "hello", + metadata: {}, + customPayload: customPayload + }, + ], + ); + + // Verify that the Qdrant client's upsert method was called with the correct arguments + expect(client.upsert).toHaveBeenCalledTimes(1); + expect(client.upsert).toHaveBeenCalledWith("documents", { + wait: true, + points: [ + expect.objectContaining({ + payload: expect.objectContaining({ + content: "hello", + metadata: {}, + ...customPayload, + }), + }), + ], + }); +}); From 6d830832d5296fbc4141c76a8bc024a75618ff53 Mon Sep 17 00:00:00 2001 From: dom_ Date: Wed, 29 Nov 2023 17:15:56 -0500 Subject: [PATCH 3/9] Resolve comments in PR --- langchain/src/vectorstores/qdrant.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/langchain/src/vectorstores/qdrant.ts b/langchain/src/vectorstores/qdrant.ts index 9ff550cd49c2..51422be5d373 100644 --- a/langchain/src/vectorstores/qdrant.ts +++ b/langchain/src/vectorstores/qdrant.ts @@ -6,6 +6,7 @@ import { Embeddings } from "../embeddings/base.js"; import { VectorStore } from "./base.js"; import { Document } from "../document.js"; import { getEnvironmentVariable } from "../util/env.js"; +import { custom } from "zod"; /** * Interface for the arguments that can be passed to the @@ -125,8 +126,7 @@ export class QdrantVectorStore extends VectorStore { payload: { content: documents[idx].pageContent, metadata: documents[idx].metadata, - ...(documents[idx].customPayload ? documents[idx].customPayload != customPayload ? - documents[idx].customPayload : customPayload : customPayload), + ...(documents[idx].customPayload ?? customPayload), }, })); @@ -224,7 +224,7 @@ export class QdrantVectorStore extends VectorStore { metadatas: object[] | object, embeddings: Embeddings, dbConfig: QdrantLibArgs, - customPayload? : object + customPayload?: object ): Promise { const docs = []; for (let i = 0; i < texts.length; i += 1) { @@ -232,7 +232,7 @@ export class QdrantVectorStore extends VectorStore { const newDoc = new Document({ pageContent: texts[i], metadata, - customPayload: customPayload, + customPayload, }); docs.push(newDoc); } From a378f35a43f07964bafc2e145fffd2b01bff7f00 Mon Sep 17 00:00:00 2001 From: dom_ Date: Wed, 29 Nov 2023 17:40:26 -0500 Subject: [PATCH 4/9] Add document changes to langchain-core --- langchain-core/src/documents/document.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/langchain-core/src/documents/document.ts b/langchain-core/src/documents/document.ts index 3647d51609ff..b2ef023d6bdc 100644 --- a/langchain-core/src/documents/document.ts +++ b/langchain-core/src/documents/document.ts @@ -5,6 +5,7 @@ export interface DocumentInput< pageContent: string; metadata?: Metadata; + customPayload?: object; } /** @@ -19,6 +20,8 @@ export class Document< metadata: Metadata; + customPayload?: object | undefined; + constructor(fields: DocumentInput) { this.pageContent = fields.pageContent ? fields.pageContent.toString() From 656f57a7a78020cdfd19966fa1340b69a1ae20eb Mon Sep 17 00:00:00 2001 From: dom_ Date: Wed, 29 Nov 2023 19:24:19 -0500 Subject: [PATCH 5/9] Test because all yarn test cases are passing --- langchain-core/src/documents/document.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/langchain-core/src/documents/document.ts b/langchain-core/src/documents/document.ts index b2ef023d6bdc..a857d5aece6c 100644 --- a/langchain-core/src/documents/document.ts +++ b/langchain-core/src/documents/document.ts @@ -5,6 +5,7 @@ export interface DocumentInput< pageContent: string; metadata?: Metadata; + // Custom payload for vectorstores for an extra parameter to filter for customPayload?: object; } From 78ef653d015b5457316c3df83bcf3532e5dd5496 Mon Sep 17 00:00:00 2001 From: dom_ Date: Thu, 30 Nov 2023 08:30:56 -0500 Subject: [PATCH 6/9] Update tests and fix yarn lint --- .../src/document_loaders/tests/csv-blob.test.ts | 2 ++ .../src/document_loaders/tests/json-blob.test.ts | 4 ++++ .../document_loaders/tests/jsonl-blob.test.ts | 2 ++ langchain/src/vectorstores/qdrant.ts | 1 - langchain/src/vectorstores/tests/qdrant.test.ts | 16 +++++++--------- 5 files changed, 15 insertions(+), 10 deletions(-) diff --git a/langchain/src/document_loaders/tests/csv-blob.test.ts b/langchain/src/document_loaders/tests/csv-blob.test.ts index 44a17b904688..9f3a8d9414fe 100644 --- a/langchain/src/document_loaders/tests/csv-blob.test.ts +++ b/langchain/src/document_loaders/tests/csv-blob.test.ts @@ -46,6 +46,7 @@ test("Test CSV loader from blob", async () => { expect(docs.length).toBe(2); expect(docs[0]).toMatchInlineSnapshot(` Document { + "customPayload": undefined, "metadata": { "blobType": "text/csv", "line": 1, @@ -57,6 +58,7 @@ test("Test CSV loader from blob", async () => { `); expect(docs[1]).toMatchInlineSnapshot(` Document { + "customPayload": undefined, "metadata": { "blobType": "text/csv", "line": 2, diff --git a/langchain/src/document_loaders/tests/json-blob.test.ts b/langchain/src/document_loaders/tests/json-blob.test.ts index d12b4c988280..c9559dd3c212 100644 --- a/langchain/src/document_loaders/tests/json-blob.test.ts +++ b/langchain/src/document_loaders/tests/json-blob.test.ts @@ -39,6 +39,7 @@ test("Test JSON loader from blob", async () => { expect(docs.length).toBe(2); expect(docs[0]).toMatchInlineSnapshot(` Document { + "customPayload": undefined, "metadata": { "blobType": "application/json", "line": 1, @@ -49,6 +50,7 @@ test("Test JSON loader from blob", async () => { `); expect(docs[1]).toMatchInlineSnapshot(` Document { + "customPayload": undefined, "metadata": { "blobType": "application/json", "line": 2, @@ -87,6 +89,7 @@ test("Test JSON loader from blob", async () => { expect(docs.length).toBe(10); expect(docs[0]).toMatchInlineSnapshot(` Document { + "customPayload": undefined, "metadata": { "blobType": "application/json", "line": 1, @@ -97,6 +100,7 @@ test("Test JSON loader from blob", async () => { `); expect(docs[1]).toMatchInlineSnapshot(` Document { + "customPayload": undefined, "metadata": { "blobType": "application/json", "line": 2, diff --git a/langchain/src/document_loaders/tests/jsonl-blob.test.ts b/langchain/src/document_loaders/tests/jsonl-blob.test.ts index fbc6df040280..8cfec57fbde5 100644 --- a/langchain/src/document_loaders/tests/jsonl-blob.test.ts +++ b/langchain/src/document_loaders/tests/jsonl-blob.test.ts @@ -40,6 +40,7 @@ test("Test JSONL loader from blob", async () => { expect(docs.length).toBe(2); expect(docs[0]).toMatchInlineSnapshot(` Document { + "customPayload": undefined, "metadata": { "blobType": "application/jsonl+json", "line": 1, @@ -50,6 +51,7 @@ test("Test JSONL loader from blob", async () => { `); expect(docs[1]).toMatchInlineSnapshot(` Document { + "customPayload": undefined, "metadata": { "blobType": "application/jsonl+json", "line": 2, diff --git a/langchain/src/vectorstores/qdrant.ts b/langchain/src/vectorstores/qdrant.ts index 51422be5d373..74c3ecb4c5cf 100644 --- a/langchain/src/vectorstores/qdrant.ts +++ b/langchain/src/vectorstores/qdrant.ts @@ -6,7 +6,6 @@ import { Embeddings } from "../embeddings/base.js"; import { VectorStore } from "./base.js"; import { Document } from "../document.js"; import { getEnvironmentVariable } from "../util/env.js"; -import { custom } from "zod"; /** * Interface for the arguments that can be passed to the diff --git a/langchain/src/vectorstores/tests/qdrant.test.ts b/langchain/src/vectorstores/tests/qdrant.test.ts index edc9f8c24b62..9984acd42864 100644 --- a/langchain/src/vectorstores/tests/qdrant.test.ts +++ b/langchain/src/vectorstores/tests/qdrant.test.ts @@ -106,15 +106,13 @@ test("QdrantVectorStore adds vectors with custom payload in Document", async () }; // Add documents with custom payload - await qdrantVectorStore.addDocuments( - [ - { - pageContent: "hello", - metadata: {}, - customPayload: customPayload - }, - ], - ); + await qdrantVectorStore.addDocuments([ + { + pageContent: "hello", + metadata: {}, + customPayload, + }, + ]); // Verify that the Qdrant client's upsert method was called with the correct arguments expect(client.upsert).toHaveBeenCalledTimes(1); From e996840888b0148bcd9d00dfef06970c0bdd8d4f Mon Sep 17 00:00:00 2001 From: dom_ Date: Thu, 30 Nov 2023 16:59:37 -0500 Subject: [PATCH 7/9] Remove payload from Document and add object[] --- langchain-core/src/documents/document.ts | 4 - .../document_loaders/tests/csv-blob.test.ts | 2 - .../document_loaders/tests/json-blob.test.ts | 4 - .../document_loaders/tests/jsonl-blob.test.ts | 2 - langchain/src/vectorstores/qdrant.ts | 18 ++-- .../src/vectorstores/tests/qdrant.test.ts | 92 ++++++++++++++++--- 6 files changed, 92 insertions(+), 30 deletions(-) diff --git a/langchain-core/src/documents/document.ts b/langchain-core/src/documents/document.ts index a857d5aece6c..3647d51609ff 100644 --- a/langchain-core/src/documents/document.ts +++ b/langchain-core/src/documents/document.ts @@ -5,8 +5,6 @@ export interface DocumentInput< pageContent: string; metadata?: Metadata; - // Custom payload for vectorstores for an extra parameter to filter for - customPayload?: object; } /** @@ -21,8 +19,6 @@ export class Document< metadata: Metadata; - customPayload?: object | undefined; - constructor(fields: DocumentInput) { this.pageContent = fields.pageContent ? fields.pageContent.toString() diff --git a/langchain/src/document_loaders/tests/csv-blob.test.ts b/langchain/src/document_loaders/tests/csv-blob.test.ts index 9f3a8d9414fe..44a17b904688 100644 --- a/langchain/src/document_loaders/tests/csv-blob.test.ts +++ b/langchain/src/document_loaders/tests/csv-blob.test.ts @@ -46,7 +46,6 @@ test("Test CSV loader from blob", async () => { expect(docs.length).toBe(2); expect(docs[0]).toMatchInlineSnapshot(` Document { - "customPayload": undefined, "metadata": { "blobType": "text/csv", "line": 1, @@ -58,7 +57,6 @@ test("Test CSV loader from blob", async () => { `); expect(docs[1]).toMatchInlineSnapshot(` Document { - "customPayload": undefined, "metadata": { "blobType": "text/csv", "line": 2, diff --git a/langchain/src/document_loaders/tests/json-blob.test.ts b/langchain/src/document_loaders/tests/json-blob.test.ts index c9559dd3c212..d12b4c988280 100644 --- a/langchain/src/document_loaders/tests/json-blob.test.ts +++ b/langchain/src/document_loaders/tests/json-blob.test.ts @@ -39,7 +39,6 @@ test("Test JSON loader from blob", async () => { expect(docs.length).toBe(2); expect(docs[0]).toMatchInlineSnapshot(` Document { - "customPayload": undefined, "metadata": { "blobType": "application/json", "line": 1, @@ -50,7 +49,6 @@ test("Test JSON loader from blob", async () => { `); expect(docs[1]).toMatchInlineSnapshot(` Document { - "customPayload": undefined, "metadata": { "blobType": "application/json", "line": 2, @@ -89,7 +87,6 @@ test("Test JSON loader from blob", async () => { expect(docs.length).toBe(10); expect(docs[0]).toMatchInlineSnapshot(` Document { - "customPayload": undefined, "metadata": { "blobType": "application/json", "line": 1, @@ -100,7 +97,6 @@ test("Test JSON loader from blob", async () => { `); expect(docs[1]).toMatchInlineSnapshot(` Document { - "customPayload": undefined, "metadata": { "blobType": "application/json", "line": 2, diff --git a/langchain/src/document_loaders/tests/jsonl-blob.test.ts b/langchain/src/document_loaders/tests/jsonl-blob.test.ts index 8cfec57fbde5..fbc6df040280 100644 --- a/langchain/src/document_loaders/tests/jsonl-blob.test.ts +++ b/langchain/src/document_loaders/tests/jsonl-blob.test.ts @@ -40,7 +40,6 @@ test("Test JSONL loader from blob", async () => { expect(docs.length).toBe(2); expect(docs[0]).toMatchInlineSnapshot(` Document { - "customPayload": undefined, "metadata": { "blobType": "application/jsonl+json", "line": 1, @@ -51,7 +50,6 @@ test("Test JSONL loader from blob", async () => { `); expect(docs[1]).toMatchInlineSnapshot(` Document { - "customPayload": undefined, "metadata": { "blobType": "application/jsonl+json", "line": 2, diff --git a/langchain/src/vectorstores/qdrant.ts b/langchain/src/vectorstores/qdrant.ts index 74c3ecb4c5cf..3643f1f70817 100644 --- a/langchain/src/vectorstores/qdrant.ts +++ b/langchain/src/vectorstores/qdrant.ts @@ -89,7 +89,7 @@ export class QdrantVectorStore extends VectorStore { */ async addDocuments( documents: Document[], - customPayload?: object + customPayload?: object[] ): Promise { const texts = documents.map(({ pageContent }) => pageContent); await this.addVectors( @@ -111,7 +111,7 @@ export class QdrantVectorStore extends VectorStore { async addVectors( vectors: number[][], documents: Document[], - customPayload?: object + customPayload?: object[] ): Promise { if (vectors.length === 0) { return; @@ -125,7 +125,7 @@ export class QdrantVectorStore extends VectorStore { payload: { content: documents[idx].pageContent, metadata: documents[idx].metadata, - ...(documents[idx].customPayload ?? customPayload), + customPayload: customPayload?.[idx], }, })); @@ -223,7 +223,7 @@ export class QdrantVectorStore extends VectorStore { metadatas: object[] | object, embeddings: Embeddings, dbConfig: QdrantLibArgs, - customPayload?: object + customPayload?: object[] ): Promise { const docs = []; for (let i = 0; i < texts.length; i += 1) { @@ -231,11 +231,15 @@ export class QdrantVectorStore extends VectorStore { const newDoc = new Document({ pageContent: texts[i], metadata, - customPayload, }); docs.push(newDoc); } - return QdrantVectorStore.fromDocuments(docs, embeddings, dbConfig); + return QdrantVectorStore.fromDocuments( + docs, + embeddings, + dbConfig, + customPayload + ); } /** @@ -251,7 +255,7 @@ export class QdrantVectorStore extends VectorStore { docs: Document[], embeddings: Embeddings, dbConfig: QdrantLibArgs, - customPayload?: object + customPayload?: object[] ): Promise { const instance = new this(embeddings, dbConfig); await instance.addDocuments(docs, customPayload); diff --git a/langchain/src/vectorstores/tests/qdrant.test.ts b/langchain/src/vectorstores/tests/qdrant.test.ts index 9984acd42864..9448ed8dc361 100644 --- a/langchain/src/vectorstores/tests/qdrant.test.ts +++ b/langchain/src/vectorstores/tests/qdrant.test.ts @@ -50,10 +50,12 @@ test("QdrantVectorStore adds vectors with custom payload", async () => { }); // Define a custom payload - const customPayload = { - customField1: "value1", - customField2: "value2", - }; + const customPayload = [ + { + customField1: "value1", + customField2: "value2", + }, + ]; // Add documents with custom payload await qdrantVectorStore.addDocuments( @@ -75,14 +77,14 @@ test("QdrantVectorStore adds vectors with custom payload", async () => { payload: expect.objectContaining({ content: "hello", metadata: {}, - ...customPayload, + customPayload: customPayload[0], }), }), ], }); }); -test("QdrantVectorStore adds vectors with custom payload in Document", async () => { +test("QdrantVectorStore adds vectors with multiple custom payload", async () => { // Mock Qdrant client const client = { upsert: jest.fn(), @@ -100,17 +102,86 @@ test("QdrantVectorStore adds vectors with custom payload in Document", async () }); // Define a custom payload - const customPayload = { - customField1: "value1", - customField2: "value2", + const customPayload = [ + { + customField1: "value1", + customField2: "value2", + }, + { + customField3: "value3", + }, + ]; + + // Add documents with custom payload + await qdrantVectorStore.addDocuments( + [ + { + pageContent: "hello", + metadata: {}, + }, + { + pageContent: "Goodbye", + metadata: {}, + }, + { + pageContent: "D01", + metadata: {}, + }, + ], + customPayload + ); + + // Verify that the Qdrant client's upsert method was called with the correct arguments + expect(client.upsert).toHaveBeenCalledTimes(1); + expect(client.upsert).toHaveBeenCalledWith("documents", { + wait: true, + points: [ + expect.objectContaining({ + payload: expect.objectContaining({ + content: "hello", + metadata: {}, + customPayload: customPayload[0], + }), + }), + expect.objectContaining({ + payload: expect.objectContaining({ + content: "Goodbye", + metadata: {}, + customPayload: customPayload[1], + }), + }), + expect.objectContaining({ + payload: expect.objectContaining({ + content: "D01", + metadata: {}, + }), + }), + ], + }); +}); + +test("QdrantVectorStore adds vectors with no custom payload", async () => { + // Mock Qdrant client + const client = { + upsert: jest.fn(), + search: jest.fn().mockResolvedValue([]), + getCollections: jest.fn().mockResolvedValue({ collections: [] }), + createCollection: jest.fn(), }; + // Mock embeddings + const embeddings = new FakeEmbeddings(); + + // Create QdrantVectorStore instance with the mock client + const qdrantVectorStore = new QdrantVectorStore(embeddings, { + client: client as any, + }); + // Add documents with custom payload await qdrantVectorStore.addDocuments([ { pageContent: "hello", metadata: {}, - customPayload, }, ]); @@ -123,7 +194,6 @@ test("QdrantVectorStore adds vectors with custom payload in Document", async () payload: expect.objectContaining({ content: "hello", metadata: {}, - ...customPayload, }), }), ], From d7f2e534db3c283282f76ac51422c4a5a16e65b5 Mon Sep 17 00:00:00 2001 From: dom_ Date: Wed, 6 Dec 2023 09:28:55 -0500 Subject: [PATCH 8/9] Remove object[] and replace with objects +comments --- langchain/src/vectorstores/qdrant.ts | 38 ++++++++--------- .../src/vectorstores/tests/qdrant.test.ts | 41 +++++++++++-------- 2 files changed, 42 insertions(+), 37 deletions(-) diff --git a/langchain/src/vectorstores/qdrant.ts b/langchain/src/vectorstores/qdrant.ts index 3643f1f70817..4bdca9722db9 100644 --- a/langchain/src/vectorstores/qdrant.ts +++ b/langchain/src/vectorstores/qdrant.ts @@ -2,6 +2,7 @@ import { QdrantClient } from "@qdrant/js-client-rest"; import type { Schemas as QdrantSchemas } from "@qdrant/js-client-rest"; import { v4 as uuid } from "uuid"; +import { AddDocumentOptions } from "closevector-common"; import { Embeddings } from "../embeddings/base.js"; import { VectorStore } from "./base.js"; import { Document } from "../document.js"; @@ -19,6 +20,7 @@ export interface QdrantLibArgs { apiKey?: string; collectionName?: string; collectionConfig?: QdrantSchemas["CreateCollection"]; + customPayload?: object[]; } /** @@ -84,18 +86,18 @@ export class QdrantVectorStore extends VectorStore { * from the documents using the `Embeddings` instance and then adds the * vectors to the database. * @param documents Array of `Document` instances to be added to the Qdrant database. - * @param customPayload Optional 'object' in JSON format used to query database on more criteria + * @param documentOptions Optional `AddDocumentOptions` which has a list of JSON objects for extra querying * @returns Promise that resolves when the documents have been added to the database. */ async addDocuments( documents: Document[], - customPayload?: object[] + documentOptions?: AddDocumentOptions ): Promise { const texts = documents.map(({ pageContent }) => pageContent); await this.addVectors( await this.embeddings.embedDocuments(texts), documents, - customPayload + documentOptions ); } @@ -105,13 +107,13 @@ export class QdrantVectorStore extends VectorStore { * database. * @param vectors Array of vectors to be added to the Qdrant database. * @param documents Array of `Document` instances associated with the vectors. - * @param customPayload Optional 'object' in JSON format used to query database on more criteria + * @param documentOptions Optional `AddDocumentOptions` which has a list of JSON objects for extra querying * @returns Promise that resolves when the vectors have been added to the database. */ async addVectors( vectors: number[][], documents: Document[], - customPayload?: object[] + documentOptions?: AddDocumentOptions ): Promise { if (vectors.length === 0) { return; @@ -125,7 +127,7 @@ export class QdrantVectorStore extends VectorStore { payload: { content: documents[idx].pageContent, metadata: documents[idx].metadata, - customPayload: customPayload?.[idx], + customPayload: documentOptions?.customPayload[idx], }, })); @@ -215,15 +217,13 @@ export class QdrantVectorStore extends VectorStore { * @param metadatas Array or single object of metadata to be associated with the texts. * @param embeddings `Embeddings` instance used to generate vectors from the texts. * @param dbConfig `QdrantLibArgs` instance specifying the configuration for the Qdrant database. - * @param customPayload Optional 'object' in JSON format used to query database on more criteria * @returns Promise that resolves with a new `QdrantVectorStore` instance. */ static async fromTexts( texts: string[], metadatas: object[] | object, embeddings: Embeddings, - dbConfig: QdrantLibArgs, - customPayload?: object[] + dbConfig: QdrantLibArgs ): Promise { const docs = []; for (let i = 0; i < texts.length; i += 1) { @@ -234,12 +234,7 @@ export class QdrantVectorStore extends VectorStore { }); docs.push(newDoc); } - return QdrantVectorStore.fromDocuments( - docs, - embeddings, - dbConfig, - customPayload - ); + return QdrantVectorStore.fromDocuments(docs, embeddings, dbConfig); } /** @@ -248,17 +243,22 @@ export class QdrantVectorStore extends VectorStore { * @param docs Array of `Document` instances to be added to the Qdrant database. * @param embeddings `Embeddings` instance used to generate vectors from the documents. * @param dbConfig `QdrantLibArgs` instance specifying the configuration for the Qdrant database. - * @param customPayload Optional 'object' in JSON format used to query database on more criteria * @returns Promise that resolves with a new `QdrantVectorStore` instance. */ static async fromDocuments( docs: Document[], embeddings: Embeddings, - dbConfig: QdrantLibArgs, - customPayload?: object[] + dbConfig: QdrantLibArgs ): Promise { const instance = new this(embeddings, dbConfig); - await instance.addDocuments(docs, customPayload); + if (dbConfig.customPayload) { + const documentOptions = { + customPayload: dbConfig?.customPayload, + } as AddDocumentOptions; + await instance.addDocuments(docs, documentOptions); + } else { + await instance.addDocuments(docs); + } return instance; } diff --git a/langchain/src/vectorstores/tests/qdrant.test.ts b/langchain/src/vectorstores/tests/qdrant.test.ts index 9448ed8dc361..4e7cfcdac590 100644 --- a/langchain/src/vectorstores/tests/qdrant.test.ts +++ b/langchain/src/vectorstores/tests/qdrant.test.ts @@ -1,5 +1,6 @@ /* eslint-disable @typescript-eslint/no-explicit-any */ import { jest, test, expect } from "@jest/globals"; +import { AddDocumentOptions } from "closevector-common"; import { FakeEmbeddings } from "../../embeddings/fake.js"; import { QdrantVectorStore } from "../qdrant.js"; @@ -50,12 +51,14 @@ test("QdrantVectorStore adds vectors with custom payload", async () => { }); // Define a custom payload - const customPayload = [ - { - customField1: "value1", - customField2: "value2", - }, - ]; + const customPayload = { + customPayload: [ + { + customField1: "value1", + customField2: "value2", + }, + ], + } as AddDocumentOptions; // Add documents with custom payload await qdrantVectorStore.addDocuments( @@ -77,7 +80,7 @@ test("QdrantVectorStore adds vectors with custom payload", async () => { payload: expect.objectContaining({ content: "hello", metadata: {}, - customPayload: customPayload[0], + customPayload: customPayload.customPayload[0], }), }), ], @@ -102,15 +105,17 @@ test("QdrantVectorStore adds vectors with multiple custom payload", async () => }); // Define a custom payload - const customPayload = [ - { - customField1: "value1", - customField2: "value2", - }, - { - customField3: "value3", - }, - ]; + const customPayload = { + customPayload: [ + { + customField1: "value1", + customField2: "value2", + }, + { + customField3: "value3", + }, + ], + } as AddDocumentOptions; // Add documents with custom payload await qdrantVectorStore.addDocuments( @@ -140,14 +145,14 @@ test("QdrantVectorStore adds vectors with multiple custom payload", async () => payload: expect.objectContaining({ content: "hello", metadata: {}, - customPayload: customPayload[0], + customPayload: customPayload.customPayload[0], }), }), expect.objectContaining({ payload: expect.objectContaining({ content: "Goodbye", metadata: {}, - customPayload: customPayload[1], + customPayload: customPayload.customPayload[1], }), }), expect.objectContaining({ From df87af2c07e794bd7ef1f2ab0be3e54037fd20be Mon Sep 17 00:00:00 2001 From: jacoblee93 Date: Tue, 12 Dec 2023 09:13:11 -0800 Subject: [PATCH 9/9] Update add document types --- libs/langchain-community/src/vectorstores/qdrant.ts | 2 +- .../langchain-community/src/vectorstores/tests/qdrant.test.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/libs/langchain-community/src/vectorstores/qdrant.ts b/libs/langchain-community/src/vectorstores/qdrant.ts index 718211ba911a..8e72426281dc 100644 --- a/libs/langchain-community/src/vectorstores/qdrant.ts +++ b/libs/langchain-community/src/vectorstores/qdrant.ts @@ -281,4 +281,4 @@ export class QdrantVectorStore extends VectorStore { await instance.ensureCollection(); return instance; } -} \ No newline at end of file +} diff --git a/libs/langchain-community/src/vectorstores/tests/qdrant.test.ts b/libs/langchain-community/src/vectorstores/tests/qdrant.test.ts index 0c4ca0becad9..ee3fa42cf13e 100644 --- a/libs/langchain-community/src/vectorstores/tests/qdrant.test.ts +++ b/libs/langchain-community/src/vectorstores/tests/qdrant.test.ts @@ -14,9 +14,9 @@ test("QdrantVectorStore works", async () => { const embeddings = new FakeEmbeddings(); - const store = new QdrantVectorStore(embeddings, { + const store = new QdrantVectorStore(embeddings, { // eslint-disable-next-line @typescript-eslint/no-explicit-any - client: client as any + client: client as any, }); expect(store).toBeDefined();