From 8fe12f324d1bfa3dacc41535529728490aade0fe Mon Sep 17 00:00:00 2001 From: castroCrea Date: Fri, 10 Nov 2023 22:08:27 +0100 Subject: [PATCH 01/13] Add File to Open Ai assistant --- .../experimental/openai_assistant/index.ts | 3 ++ .../src/experimental/openai_files/index.ts | 26 ++++++++++++++ .../tests/openai_file.int.test.ts | 36 +++++++++++++++++++ .../experimental/openai_files/tests/test.txt | 1 + 4 files changed, 66 insertions(+) create mode 100644 langchain/src/experimental/openai_files/index.ts create mode 100644 langchain/src/experimental/openai_files/tests/openai_file.int.test.ts create mode 100644 langchain/src/experimental/openai_files/tests/test.txt diff --git a/langchain/src/experimental/openai_assistant/index.ts b/langchain/src/experimental/openai_assistant/index.ts index 6b1de22abbea..9ccd77933abb 100644 --- a/langchain/src/experimental/openai_assistant/index.ts +++ b/langchain/src/experimental/openai_assistant/index.ts @@ -60,11 +60,13 @@ export class OpenAIAssistantRunnable< clientOptions, asAgent, pollIntervalMs, + file_ids, }: Omit, "assistantId"> & { model: string; name?: string; instructions?: string; tools?: OpenAIToolType | Array; + file_ids?: string[]; }) { const formattedTools = tools?.map((tool) => { @@ -80,6 +82,7 @@ export class OpenAIAssistantRunnable< instructions, tools: formattedTools, model, + file_ids, }); return new this({ diff --git a/langchain/src/experimental/openai_files/index.ts b/langchain/src/experimental/openai_files/index.ts new file mode 100644 index 000000000000..181809c401f0 --- /dev/null +++ b/langchain/src/experimental/openai_files/index.ts @@ -0,0 +1,26 @@ +import { ClientOptions, OpenAI as OpenAIClient } from "openai"; +import fs from "fs"; + +export type OpenAIFilesCreate = { + file: fs.ReadStream; + purpose: "assistants" | "fine-tune"; +}; +export type OpenAIFilesInput = { + client?: OpenAIClient; + clientOptions?: ClientOptions; +}; + +export class OpenAIFiles { + static async create({ file, purpose }: OpenAIFilesCreate) { + const oaiClient = new OpenAIClient(); + return oaiClient.files.create({ + file, + purpose, + }); + } + + static async del({ fileId }: { fileId: string }) { + const oaiClient = new OpenAIClient(); + return oaiClient.files.del(fileId); + } +} diff --git a/langchain/src/experimental/openai_files/tests/openai_file.int.test.ts b/langchain/src/experimental/openai_files/tests/openai_file.int.test.ts new file mode 100644 index 000000000000..93b16aeade01 --- /dev/null +++ b/langchain/src/experimental/openai_files/tests/openai_file.int.test.ts @@ -0,0 +1,36 @@ +import * as fs from "fs"; +import * as path from "path"; +import { OpenAIFiles } from "../index.js"; + +test("Send a file to Open AI", async () => { + const file = await OpenAIFiles.create({ + file: fs.createReadStream(path.resolve(__dirname, `./test.txt`)), + purpose: "assistants", + }); + console.log(file); + expect(file.id).toBeDefined(); + expect(file.object).toBe("file"); + /** + * Output + { + "id": "file-BK7bzQj3FfZFXr7DbL6xJwfo", + "object": "file", + "bytes": 120000, + "created_at": 1677610602, + "filename": "salesOverview.pdf", + "purpose": "assistants", + } + */ + const result = await OpenAIFiles.del({ fileId: file.id }); + console.log(result); + expect(result.id).toBe(file.id); + expect(result.deleted).toBeTruthy(); + /** + * Output: + { + "id": "file-abc123", + "object": "file", + "deleted": true + } + */ +}); diff --git a/langchain/src/experimental/openai_files/tests/test.txt b/langchain/src/experimental/openai_files/tests/test.txt new file mode 100644 index 000000000000..70c379b63ffa --- /dev/null +++ b/langchain/src/experimental/openai_files/tests/test.txt @@ -0,0 +1 @@ +Hello world \ No newline at end of file From 52de6ce87b6872c7ab1d445f692a3b26165e8da8 Mon Sep 17 00:00:00 2001 From: castroCrea Date: Fri, 10 Nov 2023 22:21:10 +0100 Subject: [PATCH 02/13] =?UTF-8?q?=F0=9F=93=9D=20Add=20documentation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../agents/agent_types/openai_file.mdx | 76 +++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 docs/docs/modules/agents/agent_types/openai_file.mdx diff --git a/docs/docs/modules/agents/agent_types/openai_file.mdx b/docs/docs/modules/agents/agent_types/openai_file.mdx new file mode 100644 index 000000000000..c170fc4932ca --- /dev/null +++ b/docs/docs/modules/agents/agent_types/openai_file.mdx @@ -0,0 +1,76 @@ +# OpenAI File + +:::info +The [OpenAI File is used in OpenAI Assistant API](https://platform.openai.com/docs/assistants/how-it-works/creating-assistants) is still in beta. +[API reference](https://platform.openai.com/docs/api-reference/files/object) +::: + +Files are used to upload documents that can be used with features like Assistants and Fine-tuning. + +We've implemented the File API in LangChain with create and delete. + +## Creating an File + +```typescript +import * as fs from "fs"; +import * as path from "path"; +import { OpenAIFiles } from "langchain/experimental/openai_files"; + +const file = await OpenAIFiles.create({ + file: fs.createReadStream(path.resolve(__dirname, `./test.txt`)), + purpose: "assistants", +}); +/** + * Output + { + "id": "file-BK7bzQj3FfZFXr7DbL6xJwfo", + "object": "file", + "bytes": 120000, + "created_at": 1677610602, + "filename": "salesOverview.pdf", + "purpose": "assistants", + } + */ +``` + +## Use File in AI Assistant + +```typescript +import * as fs from "fs"; +import * as path from "path"; +import { OpenAIAssistants } from "langchain/experimental/openai_assistants"; +import { OpenAIFiles } from "langchain/experimental/openai_files"; + +const file = await OpenAIFiles.create({ + file: fs.createReadStream(path.resolve(__dirname, `./test.txt`)), + purpose: "assistants", +}); + +const agent = await OpenAIAssistantRunnable.createAssistant({ + model: "gpt-3.5-turbo-1106", + instructions: + "You are a weather bot. Use the provided functions to answer questions.", + name: "Weather Assistant", + tools, + asAgent: true, + file_ids: [file.id], +}); +``` + +## Delete an File + +```typescript +import * as fs from "fs"; +import * as path from "path"; +import { OpenAIFiles } from "langchain/experimental/openai_files"; + +const result = await OpenAIFiles.del({ fileId: file.id }); +/** + * Output: + { + "id": "file-abc123", + "object": "file", + "deleted": true + } + */ +``` From 9534c0d17e6f58d87fb60e97b0fb8a51b4014647 Mon Sep 17 00:00:00 2001 From: castroCrea Date: Mon, 13 Nov 2023 12:55:54 +0100 Subject: [PATCH 03/13] =?UTF-8?q?=E2=9C=A8=20Add=20Open=20File=20API?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../experimental/openai_assistant/index.ts | 6 +- .../src/experimental/openai_files/index.ts | 74 +++++++++++++++++-- 2 files changed, 69 insertions(+), 11 deletions(-) diff --git a/langchain/src/experimental/openai_assistant/index.ts b/langchain/src/experimental/openai_assistant/index.ts index 9ccd77933abb..60f95757f778 100644 --- a/langchain/src/experimental/openai_assistant/index.ts +++ b/langchain/src/experimental/openai_assistant/index.ts @@ -60,13 +60,13 @@ export class OpenAIAssistantRunnable< clientOptions, asAgent, pollIntervalMs, - file_ids, + fileIds, }: Omit, "assistantId"> & { model: string; name?: string; instructions?: string; tools?: OpenAIToolType | Array; - file_ids?: string[]; + fileIds?: string[]; }) { const formattedTools = tools?.map((tool) => { @@ -82,7 +82,7 @@ export class OpenAIAssistantRunnable< instructions, tools: formattedTools, model, - file_ids, + file_ids: fileIds, }); return new this({ diff --git a/langchain/src/experimental/openai_files/index.ts b/langchain/src/experimental/openai_files/index.ts index 181809c401f0..22c5ef2da0a3 100644 --- a/langchain/src/experimental/openai_files/index.ts +++ b/langchain/src/experimental/openai_files/index.ts @@ -1,17 +1,25 @@ -import { ClientOptions, OpenAI as OpenAIClient } from "openai"; -import fs from "fs"; +import { OpenAI as OpenAIClient } from "openai"; +import fs from "node:fs"; +import { RequestOptions } from "openai/core.mjs"; export type OpenAIFilesCreate = { file: fs.ReadStream; purpose: "assistants" | "fine-tune"; }; -export type OpenAIFilesInput = { - client?: OpenAIClient; - clientOptions?: ClientOptions; -}; export class OpenAIFiles { - static async create({ file, purpose }: OpenAIFilesCreate) { + /** + * Upload file + * Upload a file that can be used across various endpoints. The size of all the files uploaded by one organization can be up to 100 GB. + * + * The size of individual files can be a maximum of 512 MB. See the Assistants Tools guide to learn more about the types of files supported. The Fine-tuning API only supports .jsonl files. + * + * @link https://platform.openai.com/docs/api-reference/files/create + * @param file: fs.ReadStream + * @param purpose: "assistants" | "fine-tune" + * @returns + */ + static async createFile({ file, purpose }: OpenAIFilesCreate) { const oaiClient = new OpenAIClient(); return oaiClient.files.create({ file, @@ -19,8 +27,58 @@ export class OpenAIFiles { }); } - static async del({ fileId }: { fileId: string }) { + /** + * Delete a file. + * @link https://platform.openai.com/docs/api-reference/files/delete + * + * @param fileId: string + * @returns + */ + static async deleteFile({ fileId }: { fileId: string }) { const oaiClient = new OpenAIClient(); return oaiClient.files.del(fileId); } + + /** + * List files + * Returns a list of files that belong to the user's organization. + * @link https://platform.openai.com/docs/api-reference/files/list + * @param query: OpenAIClient.Files.FileListParams | undefined + * @param options: RequestOptions | undefined + * @returns + */ + static async listFiles(props?: { + query?: OpenAIClient.Files.FileListParams; + options?: RequestOptions; + }) { + const oaiClient = new OpenAIClient(); + return oaiClient.files.list(props?.query, props?.options); + } + + /** + * Retrieve file + * Returns information about a specific file. + * @link https://platform.openai.com/docs/api-reference/files/retrieve + * @param fileId: string + * @returns + */ + static async retrieveFile({ fileId }: { fileId: string }) { + const oaiClient = new OpenAIClient(); + return oaiClient.files.retrieve(fileId); + } + + /** + * Retrieve file content + * Returns the contents of the specified file. + * + * Note: You can't retrieve the contents of a file that was uploaded with the "purpose": "assistants" API. + * + * @link https://platform.openai.com/docs/api-reference/files/retrieve-contents + * @param fileId: string + * @returns + */ + static async retrieveFileContent({ fileId }: { fileId: string }) { + const oaiClient = new OpenAIClient(); + return oaiClient.files.retrieveContent(fileId); + } } From bf8f271fe82dbceb008c4368055e605046f10265 Mon Sep 17 00:00:00 2001 From: castroCrea Date: Mon, 13 Nov 2023 12:56:17 +0100 Subject: [PATCH 04/13] =?UTF-8?q?=F0=9F=93=9D=20Add=20documentation=20on?= =?UTF-8?q?=20Open=20AI=20File=20API?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../agents/agent_types/openai_file.mdx | 140 +++++++++++++++--- 1 file changed, 118 insertions(+), 22 deletions(-) diff --git a/docs/docs/modules/agents/agent_types/openai_file.mdx b/docs/docs/modules/agents/agent_types/openai_file.mdx index c170fc4932ca..07d03c3561bb 100644 --- a/docs/docs/modules/agents/agent_types/openai_file.mdx +++ b/docs/docs/modules/agents/agent_types/openai_file.mdx @@ -9,7 +9,24 @@ Files are used to upload documents that can be used with features like Assistant We've implemented the File API in LangChain with create and delete. -## Creating an File +The `File` object represents a document that has been uploaded to OpenAI. + +``` +{ + "id": "file-abc123", + "object": "file", + "bytes": 120000, + "created_at": 1677610602, + "filename": "salesOverview.pdf", + "purpose": "assistants", +} +``` + +## Create a File + +Upload a file that can be used across various endpoints. The size of all the files uploaded by one organization can be up to **100 GB**. + +The size of individual files can be a maximum of **512 MB**. See the Assistants Tools guide to learn more about the types of files supported. The Fine-tuning API only supports `.jsonl` files. ```typescript import * as fs from "fs"; @@ -21,16 +38,16 @@ const file = await OpenAIFiles.create({ purpose: "assistants", }); /** - * Output - { - "id": "file-BK7bzQj3FfZFXr7DbL6xJwfo", - "object": "file", - "bytes": 120000, - "created_at": 1677610602, - "filename": "salesOverview.pdf", - "purpose": "assistants", - } - */ +* Output + { + "id": "file-BK7bzQj3FfZFXr7DbL6xJwfo", + "object": "file", + "bytes": 120000, + "created_at": 1677610602, + "filename": "salesOverview.pdf", + "purpose": "assistants", + } +*/ ``` ## Use File in AI Assistant @@ -38,10 +55,10 @@ const file = await OpenAIFiles.create({ ```typescript import * as fs from "fs"; import * as path from "path"; -import { OpenAIAssistants } from "langchain/experimental/openai_assistants"; +import { OpenAIAssistantRunnable } from "langchain/experimental/openai_assistant"; import { OpenAIFiles } from "langchain/experimental/openai_files"; -const file = await OpenAIFiles.create({ +const file = await OpenAIFiles.createFile({ file: fs.createReadStream(path.resolve(__dirname, `./test.txt`)), purpose: "assistants", }); @@ -57,20 +74,99 @@ const agent = await OpenAIAssistantRunnable.createAssistant({ }); ``` -## Delete an File +## Delete a File + +Delete a file. + +```typescript +import * as fs from "fs"; +import * as path from "path"; +import { OpenAIFiles } from "langchain/experimental/openai_files"; + +const result = await OpenAIFiles.deleteFile({ fileId: file.id }); +/** +* Output: + { + "id": "file-abc123", + "object": "file", + "deleted": true + } +*/ +``` + +## List all Files + +Returns a list of files that belong to the user's organization. + +`purpose`?: string +Only return files with the given purpose. ```typescript import * as fs from "fs"; import * as path from "path"; import { OpenAIFiles } from "langchain/experimental/openai_files"; -const result = await OpenAIFiles.del({ fileId: file.id }); +const result = await OpenAIFiles.listFiles({ purpose: "assistants" }); /** - * Output: - { - "id": "file-abc123", - "object": "file", - "deleted": true - } - */ +* Output: + { + "data": [ + { + "id": "file-abc123", + "object": "file", + "bytes": 175, + "created_at": 1613677385, + "filename": "salesOverview.pdf", + "purpose": "assistants", + }, + { + "id": "file-abc123", + "object": "file", + "bytes": 140, + "created_at": 1613779121, + "filename": "puppy.jsonl", + "purpose": "fine-tune", + } + ], + "object": "list" + } +*/ +``` + +## Retrieve File + +Returns information about a specific file. + +```typescript +import * as fs from "fs"; +import * as path from "path"; +import { OpenAIFiles } from "langchain/experimental/openai_files"; + +const result = await OpenAIFiles.retrieveFile({ fileId: file.id }); +/** +* Output: + { + "id": "file-abc123", + "object": "file", + "bytes": 120000, + "created_at": 1677610602, + "filename": "mydata.jsonl", + "purpose": "fine-tune", + } +*/ +``` + +## Retrieve File Content + +Returns the contents of the specified file. + +You can't retrieve the contents of a file that was uploaded with the "purpose": "assistants" API. + +```typescript +import * as fs from "fs"; +import * as path from "path"; +import { OpenAIFiles } from "langchain/experimental/openai_files"; + +const result = await OpenAIFiles.retrieveFileContent({ fileId: file.id }); +// Return the file content. ``` From 4488fc61848c011783bc135eadd1d49c7524b7ac Mon Sep 17 00:00:00 2001 From: castroCrea Date: Mon, 13 Nov 2023 12:56:28 +0100 Subject: [PATCH 05/13] =?UTF-8?q?=E2=9C=85=20Add=20test=20on=20Open=20AI?= =?UTF-8?q?=20File=20API?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../tests/openai_file.int.test.ts | 66 +++++++++++++++++-- .../openai_files/tests/test.jsonl | 1 + .../experimental/openai_files/tests/test.txt | 1 - 3 files changed, 60 insertions(+), 8 deletions(-) create mode 100644 langchain/src/experimental/openai_files/tests/test.jsonl delete mode 100644 langchain/src/experimental/openai_files/tests/test.txt diff --git a/langchain/src/experimental/openai_files/tests/openai_file.int.test.ts b/langchain/src/experimental/openai_files/tests/openai_file.int.test.ts index 93b16aeade01..956bd80d4963 100644 --- a/langchain/src/experimental/openai_files/tests/openai_file.int.test.ts +++ b/langchain/src/experimental/openai_files/tests/openai_file.int.test.ts @@ -1,13 +1,20 @@ import * as fs from "fs"; import * as path from "path"; +import { fileURLToPath } from "url"; +import { dirname } from "path"; import { OpenAIFiles } from "../index.js"; -test("Send a file to Open AI", async () => { - const file = await OpenAIFiles.create({ - file: fs.createReadStream(path.resolve(__dirname, `./test.txt`)), - purpose: "assistants", +/** + * Otherwise we got the error __dirname doesn't exist + */ +const __filename = fileURLToPath(import.meta.url); +const __dirname = dirname(__filename); + +test("Use file with Open AI", async () => { + const file = await OpenAIFiles.createFile({ + file: fs.createReadStream(path.resolve(__dirname, `./test.jsonl`)), + purpose: "fine-tune", }); - console.log(file); expect(file.id).toBeDefined(); expect(file.object).toBe("file"); /** @@ -21,8 +28,53 @@ test("Send a file to Open AI", async () => { "purpose": "assistants", } */ - const result = await OpenAIFiles.del({ fileId: file.id }); - console.log(result); + const fileContent = await OpenAIFiles.retrieveFileContent({ + fileId: file.id, + }); + console.log(fileContent); + expect(fileContent).toBeDefined(); + /** + * Output + { + "id": "file-BK7bzQj3FfZFXr7DbL6xJwfo", + "object": "file", + "bytes": 120000, + "created_at": 1677610602, + "filename": "salesOverview.pdf", + "purpose": "assistants", + } + */ + const retrievedFile = await OpenAIFiles.retrieveFile({ + fileId: file.id, + }); + expect(retrievedFile.id).toBeDefined(); + expect(retrievedFile.object).toBe("file"); + /** + * Output + { + "id": "file-BK7bzQj3FfZFXr7DbL6xJwfo", + "object": "file", + "bytes": 120000, + "created_at": 1677610602, + "filename": "salesOverview.pdf", + "purpose": "assistants", + } + */ + const list = await OpenAIFiles.listFiles(); + expect(list).toBeDefined(); + expect(!!list.data.find((f) => f.id === file.id)).toBeTruthy(); + /** + * Output + { + "id": "file-BK7bzQj3FfZFXr7DbL6xJwfo", + "object": "file", + "bytes": 120000, + "created_at": 1677610602, + "filename": "salesOverview.pdf", + "purpose": "assistants", + } + */ + const result = await OpenAIFiles.deleteFile({ fileId: file.id }); expect(result.id).toBe(file.id); expect(result.deleted).toBeTruthy(); /** diff --git a/langchain/src/experimental/openai_files/tests/test.jsonl b/langchain/src/experimental/openai_files/tests/test.jsonl new file mode 100644 index 000000000000..cb98632f423b --- /dev/null +++ b/langchain/src/experimental/openai_files/tests/test.jsonl @@ -0,0 +1 @@ +{"messages": [{"role": "system", "content": "Marv is a factual chatbot that is also sarcastic."}, {"role": "user", "content": "What's the capital of France?"}, {"role": "assistant", "content": "Paris, as if everyone doesn't know that already."}]} diff --git a/langchain/src/experimental/openai_files/tests/test.txt b/langchain/src/experimental/openai_files/tests/test.txt deleted file mode 100644 index 70c379b63ffa..000000000000 --- a/langchain/src/experimental/openai_files/tests/test.txt +++ /dev/null @@ -1 +0,0 @@ -Hello world \ No newline at end of file From a72386f22f6d140b6fd7835eb7e4a1e07a67bb65 Mon Sep 17 00:00:00 2001 From: Brace Sproul Date: Mon, 13 Nov 2023 09:51:07 -0800 Subject: [PATCH 06/13] Update jsdoc types for params --- .../src/experimental/openai_files/index.ts | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/langchain/src/experimental/openai_files/index.ts b/langchain/src/experimental/openai_files/index.ts index 22c5ef2da0a3..0b159ab16c19 100644 --- a/langchain/src/experimental/openai_files/index.ts +++ b/langchain/src/experimental/openai_files/index.ts @@ -14,9 +14,9 @@ export class OpenAIFiles { * * The size of individual files can be a maximum of 512 MB. See the Assistants Tools guide to learn more about the types of files supported. The Fine-tuning API only supports .jsonl files. * - * @link https://platform.openai.com/docs/api-reference/files/create - * @param file: fs.ReadStream - * @param purpose: "assistants" | "fine-tune" + * @link {https://platform.openai.com/docs/api-reference/files/create} + * @param {fs.ReadStream} file + * @param {"assistants" | "fine-tune"} purpose * @returns */ static async createFile({ file, purpose }: OpenAIFilesCreate) { @@ -29,9 +29,9 @@ export class OpenAIFiles { /** * Delete a file. - * @link https://platform.openai.com/docs/api-reference/files/delete + * @link {https://platform.openai.com/docs/api-reference/files/delete} * - * @param fileId: string + * @param {string} fileId * @returns */ static async deleteFile({ fileId }: { fileId: string }) { @@ -42,9 +42,9 @@ export class OpenAIFiles { /** * List files * Returns a list of files that belong to the user's organization. - * @link https://platform.openai.com/docs/api-reference/files/list - * @param query: OpenAIClient.Files.FileListParams | undefined - * @param options: RequestOptions | undefined + * @link {https://platform.openai.com/docs/api-reference/files/list} + * @param {OpenAIClient.Files.FileListParams | undefined} query + * @param {RequestOptions | undefined} options * @returns */ static async listFiles(props?: { @@ -58,8 +58,8 @@ export class OpenAIFiles { /** * Retrieve file * Returns information about a specific file. - * @link https://platform.openai.com/docs/api-reference/files/retrieve - * @param fileId: string + * @link {https://platform.openai.com/docs/api-reference/files/retrieve} + * @param {string} fileId * @returns */ static async retrieveFile({ fileId }: { fileId: string }) { @@ -71,10 +71,10 @@ export class OpenAIFiles { * Retrieve file content * Returns the contents of the specified file. * - * Note: You can't retrieve the contents of a file that was uploaded with the "purpose": "assistants" API. + * @note You can't retrieve the contents of a file that was uploaded with the "purpose": "assistants" API. * - * @link https://platform.openai.com/docs/api-reference/files/retrieve-contents - * @param fileId: string + * @link {https://platform.openai.com/docs/api-reference/files/retrieve-contents} + * @param {string} fileId * @returns */ static async retrieveFileContent({ fileId }: { fileId: string }) { From c57751810adda1fed8630815a501518413ec8123 Mon Sep 17 00:00:00 2001 From: Brace Sproul Date: Mon, 13 Nov 2023 09:53:07 -0800 Subject: [PATCH 07/13] Fix openai request options import --- langchain/src/experimental/openai_files/index.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/langchain/src/experimental/openai_files/index.ts b/langchain/src/experimental/openai_files/index.ts index 0b159ab16c19..ae3b7f609b76 100644 --- a/langchain/src/experimental/openai_files/index.ts +++ b/langchain/src/experimental/openai_files/index.ts @@ -1,6 +1,5 @@ import { OpenAI as OpenAIClient } from "openai"; import fs from "node:fs"; -import { RequestOptions } from "openai/core.mjs"; export type OpenAIFilesCreate = { file: fs.ReadStream; @@ -44,12 +43,12 @@ export class OpenAIFiles { * Returns a list of files that belong to the user's organization. * @link {https://platform.openai.com/docs/api-reference/files/list} * @param {OpenAIClient.Files.FileListParams | undefined} query - * @param {RequestOptions | undefined} options + * @param {OpenAIClient.RequestOptions | undefined} options * @returns */ static async listFiles(props?: { query?: OpenAIClient.Files.FileListParams; - options?: RequestOptions; + options?: OpenAIClient.RequestOptions; }) { const oaiClient = new OpenAIClient(); return oaiClient.files.list(props?.query, props?.options); From 19dbfc39f1696341ae46000ad52b6475c69a9d20 Mon Sep 17 00:00:00 2001 From: Brace Sproul Date: Mon, 13 Nov 2023 10:04:40 -0800 Subject: [PATCH 08/13] Extend serializable class and add return jsdoc types --- langchain/src/experimental/openai_files/index.ts | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/langchain/src/experimental/openai_files/index.ts b/langchain/src/experimental/openai_files/index.ts index ae3b7f609b76..ceae0d7434f6 100644 --- a/langchain/src/experimental/openai_files/index.ts +++ b/langchain/src/experimental/openai_files/index.ts @@ -1,12 +1,15 @@ import { OpenAI as OpenAIClient } from "openai"; import fs from "node:fs"; +import { Serializable } from "../../load/serializable.js"; export type OpenAIFilesCreate = { file: fs.ReadStream; purpose: "assistants" | "fine-tune"; }; -export class OpenAIFiles { +export class OpenAIFiles extends Serializable { + lc_namespace = ["langchain", "experimental", "open_ai_files"]; + /** * Upload file * Upload a file that can be used across various endpoints. The size of all the files uploaded by one organization can be up to 100 GB. @@ -16,7 +19,7 @@ export class OpenAIFiles { * @link {https://platform.openai.com/docs/api-reference/files/create} * @param {fs.ReadStream} file * @param {"assistants" | "fine-tune"} purpose - * @returns + * @returns {Promise} */ static async createFile({ file, purpose }: OpenAIFilesCreate) { const oaiClient = new OpenAIClient(); @@ -31,7 +34,7 @@ export class OpenAIFiles { * @link {https://platform.openai.com/docs/api-reference/files/delete} * * @param {string} fileId - * @returns + * @returns {Promise} */ static async deleteFile({ fileId }: { fileId: string }) { const oaiClient = new OpenAIClient(); @@ -44,7 +47,7 @@ export class OpenAIFiles { * @link {https://platform.openai.com/docs/api-reference/files/list} * @param {OpenAIClient.Files.FileListParams | undefined} query * @param {OpenAIClient.RequestOptions | undefined} options - * @returns + * @returns {Promise} */ static async listFiles(props?: { query?: OpenAIClient.Files.FileListParams; @@ -59,7 +62,7 @@ export class OpenAIFiles { * Returns information about a specific file. * @link {https://platform.openai.com/docs/api-reference/files/retrieve} * @param {string} fileId - * @returns + * @returns {Promise} */ static async retrieveFile({ fileId }: { fileId: string }) { const oaiClient = new OpenAIClient(); @@ -74,7 +77,7 @@ export class OpenAIFiles { * * @link {https://platform.openai.com/docs/api-reference/files/retrieve-contents} * @param {string} fileId - * @returns + * @returns {Promise} */ static async retrieveFileContent({ fileId }: { fileId: string }) { const oaiClient = new OpenAIClient(); From c120f465946ec78f57f40ebe9c15f26afedfd99e Mon Sep 17 00:00:00 2001 From: castroCrea Date: Tue, 14 Nov 2023 09:56:26 +0100 Subject: [PATCH 09/13] =?UTF-8?q?=F0=9F=94=A7=20Add=20experimental=20opena?= =?UTF-8?q?i=5Ffiles=20entrypoint.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- langchain/scripts/create-entrypoints.js | 1 + 1 file changed, 1 insertion(+) diff --git a/langchain/scripts/create-entrypoints.js b/langchain/scripts/create-entrypoints.js index 0776e592f0da..ebefa12f458f 100644 --- a/langchain/scripts/create-entrypoints.js +++ b/langchain/scripts/create-entrypoints.js @@ -294,6 +294,7 @@ const entrypoints = { // experimental "experimental/autogpt": "experimental/autogpt/index", "experimental/openai_assistant": "experimental/openai_assistant/index", + "experimental/openai_files": "experimental/openai_files/index", "experimental/babyagi": "experimental/babyagi/index", "experimental/generative_agents": "experimental/generative_agents/index", "experimental/plan_and_execute": "experimental/plan_and_execute/index", From 68b0c14321507aba04e27c3859cd7c0eff7eb721 Mon Sep 17 00:00:00 2001 From: castroCrea Date: Tue, 14 Nov 2023 19:24:32 +0100 Subject: [PATCH 10/13] =?UTF-8?q?=F0=9F=93=9D=20Build=20the=20doc?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/api_refs/typedoc.json | 1 + environment_tests/test-exports-bun/src/entrypoints.js | 1 + environment_tests/test-exports-cf/src/entrypoints.js | 1 + environment_tests/test-exports-cjs/src/entrypoints.js | 1 + environment_tests/test-exports-esbuild/src/entrypoints.js | 1 + environment_tests/test-exports-esm/src/entrypoints.js | 1 + environment_tests/test-exports-vercel/src/entrypoints.js | 1 + environment_tests/test-exports-vite/src/entrypoints.js | 1 + langchain/.gitignore | 3 +++ langchain/package.json | 8 ++++++++ langchain/src/load/import_map.ts | 1 + 11 files changed, 20 insertions(+) diff --git a/docs/api_refs/typedoc.json b/docs/api_refs/typedoc.json index 1893a19dd55b..e13abc33f761 100644 --- a/docs/api_refs/typedoc.json +++ b/docs/api_refs/typedoc.json @@ -268,6 +268,7 @@ "../../langchain/src/util/time.ts", "../../langchain/src/experimental/autogpt/index.ts", "../../langchain/src/experimental/openai_assistant/index.ts", + "../../langchain/src/experimental/openai_files/index.ts", "../../langchain/src/experimental/babyagi/index.ts", "../../langchain/src/experimental/generative_agents/index.ts", "../../langchain/src/experimental/plan_and_execute/index.ts", diff --git a/environment_tests/test-exports-bun/src/entrypoints.js b/environment_tests/test-exports-bun/src/entrypoints.js index 116a4ceac3cd..0d6c599746bf 100644 --- a/environment_tests/test-exports-bun/src/entrypoints.js +++ b/environment_tests/test-exports-bun/src/entrypoints.js @@ -92,6 +92,7 @@ export * from "langchain/util/math"; export * from "langchain/util/time"; export * from "langchain/experimental/autogpt"; export * from "langchain/experimental/openai_assistant"; +export * from "langchain/experimental/openai_files"; export * from "langchain/experimental/babyagi"; export * from "langchain/experimental/generative_agents"; export * from "langchain/experimental/plan_and_execute"; diff --git a/environment_tests/test-exports-cf/src/entrypoints.js b/environment_tests/test-exports-cf/src/entrypoints.js index 116a4ceac3cd..0d6c599746bf 100644 --- a/environment_tests/test-exports-cf/src/entrypoints.js +++ b/environment_tests/test-exports-cf/src/entrypoints.js @@ -92,6 +92,7 @@ export * from "langchain/util/math"; export * from "langchain/util/time"; export * from "langchain/experimental/autogpt"; export * from "langchain/experimental/openai_assistant"; +export * from "langchain/experimental/openai_files"; export * from "langchain/experimental/babyagi"; export * from "langchain/experimental/generative_agents"; export * from "langchain/experimental/plan_and_execute"; diff --git a/environment_tests/test-exports-cjs/src/entrypoints.js b/environment_tests/test-exports-cjs/src/entrypoints.js index 3fe9c8a191c8..9705ca765de9 100644 --- a/environment_tests/test-exports-cjs/src/entrypoints.js +++ b/environment_tests/test-exports-cjs/src/entrypoints.js @@ -92,6 +92,7 @@ const util_math = require("langchain/util/math"); const util_time = require("langchain/util/time"); const experimental_autogpt = require("langchain/experimental/autogpt"); const experimental_openai_assistant = require("langchain/experimental/openai_assistant"); +const experimental_openai_files = require("langchain/experimental/openai_files"); const experimental_babyagi = require("langchain/experimental/babyagi"); const experimental_generative_agents = require("langchain/experimental/generative_agents"); const experimental_plan_and_execute = require("langchain/experimental/plan_and_execute"); diff --git a/environment_tests/test-exports-esbuild/src/entrypoints.js b/environment_tests/test-exports-esbuild/src/entrypoints.js index a0f8701b4049..32fabd0d55fa 100644 --- a/environment_tests/test-exports-esbuild/src/entrypoints.js +++ b/environment_tests/test-exports-esbuild/src/entrypoints.js @@ -92,6 +92,7 @@ import * as util_math from "langchain/util/math"; import * as util_time from "langchain/util/time"; import * as experimental_autogpt from "langchain/experimental/autogpt"; import * as experimental_openai_assistant from "langchain/experimental/openai_assistant"; +import * as experimental_openai_files from "langchain/experimental/openai_files"; import * as experimental_babyagi from "langchain/experimental/babyagi"; import * as experimental_generative_agents from "langchain/experimental/generative_agents"; import * as experimental_plan_and_execute from "langchain/experimental/plan_and_execute"; diff --git a/environment_tests/test-exports-esm/src/entrypoints.js b/environment_tests/test-exports-esm/src/entrypoints.js index a0f8701b4049..32fabd0d55fa 100644 --- a/environment_tests/test-exports-esm/src/entrypoints.js +++ b/environment_tests/test-exports-esm/src/entrypoints.js @@ -92,6 +92,7 @@ import * as util_math from "langchain/util/math"; import * as util_time from "langchain/util/time"; import * as experimental_autogpt from "langchain/experimental/autogpt"; import * as experimental_openai_assistant from "langchain/experimental/openai_assistant"; +import * as experimental_openai_files from "langchain/experimental/openai_files"; import * as experimental_babyagi from "langchain/experimental/babyagi"; import * as experimental_generative_agents from "langchain/experimental/generative_agents"; import * as experimental_plan_and_execute from "langchain/experimental/plan_and_execute"; diff --git a/environment_tests/test-exports-vercel/src/entrypoints.js b/environment_tests/test-exports-vercel/src/entrypoints.js index 116a4ceac3cd..0d6c599746bf 100644 --- a/environment_tests/test-exports-vercel/src/entrypoints.js +++ b/environment_tests/test-exports-vercel/src/entrypoints.js @@ -92,6 +92,7 @@ export * from "langchain/util/math"; export * from "langchain/util/time"; export * from "langchain/experimental/autogpt"; export * from "langchain/experimental/openai_assistant"; +export * from "langchain/experimental/openai_files"; export * from "langchain/experimental/babyagi"; export * from "langchain/experimental/generative_agents"; export * from "langchain/experimental/plan_and_execute"; diff --git a/environment_tests/test-exports-vite/src/entrypoints.js b/environment_tests/test-exports-vite/src/entrypoints.js index 116a4ceac3cd..0d6c599746bf 100644 --- a/environment_tests/test-exports-vite/src/entrypoints.js +++ b/environment_tests/test-exports-vite/src/entrypoints.js @@ -92,6 +92,7 @@ export * from "langchain/util/math"; export * from "langchain/util/time"; export * from "langchain/experimental/autogpt"; export * from "langchain/experimental/openai_assistant"; +export * from "langchain/experimental/openai_files"; export * from "langchain/experimental/babyagi"; export * from "langchain/experimental/generative_agents"; export * from "langchain/experimental/plan_and_execute"; diff --git a/langchain/.gitignore b/langchain/.gitignore index 648193b1d593..48effdbae811 100644 --- a/langchain/.gitignore +++ b/langchain/.gitignore @@ -748,6 +748,9 @@ experimental/autogpt.d.ts experimental/openai_assistant.cjs experimental/openai_assistant.js experimental/openai_assistant.d.ts +experimental/openai_files.cjs +experimental/openai_files.js +experimental/openai_files.d.ts experimental/babyagi.cjs experimental/babyagi.js experimental/babyagi.d.ts diff --git a/langchain/package.json b/langchain/package.json index 0905ae50f297..ad8b6c20004d 100644 --- a/langchain/package.json +++ b/langchain/package.json @@ -760,6 +760,9 @@ "experimental/openai_assistant.cjs", "experimental/openai_assistant.js", "experimental/openai_assistant.d.ts", + "experimental/openai_files.cjs", + "experimental/openai_files.js", + "experimental/openai_files.d.ts", "experimental/babyagi.cjs", "experimental/babyagi.js", "experimental/babyagi.d.ts", @@ -2648,6 +2651,11 @@ "import": "./experimental/openai_assistant.js", "require": "./experimental/openai_assistant.cjs" }, + "./experimental/openai_files": { + "types": "./experimental/openai_files.d.ts", + "import": "./experimental/openai_files.js", + "require": "./experimental/openai_files.cjs" + }, "./experimental/babyagi": { "types": "./experimental/babyagi.d.ts", "import": "./experimental/babyagi.js", diff --git a/langchain/src/load/import_map.ts b/langchain/src/load/import_map.ts index d968a29215d9..f6c5c17b957e 100644 --- a/langchain/src/load/import_map.ts +++ b/langchain/src/load/import_map.ts @@ -93,6 +93,7 @@ export * as util__math from "../util/math.js"; export * as util__time from "../util/time.js"; export * as experimental__autogpt from "../experimental/autogpt/index.js"; export * as experimental__openai_assistant from "../experimental/openai_assistant/index.js"; +export * as experimental__openai_files from "../experimental/openai_files/index.js"; export * as experimental__babyagi from "../experimental/babyagi/index.js"; export * as experimental__generative_agents from "../experimental/generative_agents/index.js"; export * as experimental__plan_and_execute from "../experimental/plan_and_execute/index.js"; From 0a85cbf6e52ad021402262ec5aa06adf92e0b4e7 Mon Sep 17 00:00:00 2001 From: castroCrea Date: Wed, 15 Nov 2023 10:55:37 +0100 Subject: [PATCH 11/13] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20Refactor=20OpenAIFil?= =?UTF-8?q?es=20to=20allow=20custom=20client?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../agents/agent_types/openai_file.mdx | 30 +++---- .../src/experimental/openai_files/index.ts | 78 ++++++++++++------- .../tests/openai_file.int.test.ts | 11 +-- 3 files changed, 70 insertions(+), 49 deletions(-) diff --git a/docs/core_docs/docs/modules/agents/agent_types/openai_file.mdx b/docs/core_docs/docs/modules/agents/agent_types/openai_file.mdx index 07d03c3561bb..8b3903b80adc 100644 --- a/docs/core_docs/docs/modules/agents/agent_types/openai_file.mdx +++ b/docs/core_docs/docs/modules/agents/agent_types/openai_file.mdx @@ -29,11 +29,10 @@ Upload a file that can be used across various endpoints. The size of all the fil The size of individual files can be a maximum of **512 MB**. See the Assistants Tools guide to learn more about the types of files supported. The Fine-tuning API only supports `.jsonl` files. ```typescript -import * as fs from "fs"; -import * as path from "path"; import { OpenAIFiles } from "langchain/experimental/openai_files"; -const file = await OpenAIFiles.create({ +const openAIFiles = new OpenAIFiles(); +const file = await openAIFiles.create({ file: fs.createReadStream(path.resolve(__dirname, `./test.txt`)), purpose: "assistants", }); @@ -53,12 +52,11 @@ const file = await OpenAIFiles.create({ ## Use File in AI Assistant ```typescript -import * as fs from "fs"; -import * as path from "path"; import { OpenAIAssistantRunnable } from "langchain/experimental/openai_assistant"; import { OpenAIFiles } from "langchain/experimental/openai_files"; -const file = await OpenAIFiles.createFile({ +const openAIFiles = new OpenAIFiles(); +const file = await openAIFiles.createFile({ file: fs.createReadStream(path.resolve(__dirname, `./test.txt`)), purpose: "assistants", }); @@ -79,11 +77,10 @@ const agent = await OpenAIAssistantRunnable.createAssistant({ Delete a file. ```typescript -import * as fs from "fs"; -import * as path from "path"; import { OpenAIFiles } from "langchain/experimental/openai_files"; -const result = await OpenAIFiles.deleteFile({ fileId: file.id }); +const openAIFiles = new OpenAIFiles(); +const result = await openAIFiles.deleteFile({ fileId: file.id }); /** * Output: { @@ -102,11 +99,10 @@ Returns a list of files that belong to the user's organization. Only return files with the given purpose. ```typescript -import * as fs from "fs"; -import * as path from "path"; import { OpenAIFiles } from "langchain/experimental/openai_files"; -const result = await OpenAIFiles.listFiles({ purpose: "assistants" }); +const openAIFiles = new OpenAIFiles(); +const result = await openAIFiles.listFiles({ purpose: "assistants" }); /** * Output: { @@ -138,11 +134,10 @@ const result = await OpenAIFiles.listFiles({ purpose: "assistants" }); Returns information about a specific file. ```typescript -import * as fs from "fs"; -import * as path from "path"; import { OpenAIFiles } from "langchain/experimental/openai_files"; -const result = await OpenAIFiles.retrieveFile({ fileId: file.id }); +const openAIFiles = new OpenAIFiles(); +const result = await openAIFiles.retrieveFile({ fileId: file.id }); /** * Output: { @@ -163,10 +158,9 @@ Returns the contents of the specified file. You can't retrieve the contents of a file that was uploaded with the "purpose": "assistants" API. ```typescript -import * as fs from "fs"; -import * as path from "path"; import { OpenAIFiles } from "langchain/experimental/openai_files"; -const result = await OpenAIFiles.retrieveFileContent({ fileId: file.id }); +const openAIFiles = new OpenAIFiles(); +const result = await openAIFiles.retrieveFileContent({ fileId: file.id }); // Return the file content. ``` diff --git a/langchain/src/experimental/openai_files/index.ts b/langchain/src/experimental/openai_files/index.ts index ceae0d7434f6..2bd244a74cde 100644 --- a/langchain/src/experimental/openai_files/index.ts +++ b/langchain/src/experimental/openai_files/index.ts @@ -1,14 +1,20 @@ -import { OpenAI as OpenAIClient } from "openai"; -import fs from "node:fs"; +import { ClientOptions, OpenAI as OpenAIClient } from "openai"; import { Serializable } from "../../load/serializable.js"; -export type OpenAIFilesCreate = { - file: fs.ReadStream; - purpose: "assistants" | "fine-tune"; +export type OpenAIFilesInput = { + client?: OpenAIClient; + clientOptions?: ClientOptions; }; export class OpenAIFiles extends Serializable { - lc_namespace = ["langchain", "experimental", "open_ai_files"]; + lc_namespace = ["langchain", "experimental"]; + + private oaiClient: OpenAIClient; + + constructor(fields?: OpenAIFilesInput) { + super(fields); + this.oaiClient = fields?.client ?? new OpenAIClient(fields?.clientOptions); + } /** * Upload file @@ -17,16 +23,19 @@ export class OpenAIFiles extends Serializable { * The size of individual files can be a maximum of 512 MB. See the Assistants Tools guide to learn more about the types of files supported. The Fine-tuning API only supports .jsonl files. * * @link {https://platform.openai.com/docs/api-reference/files/create} - * @param {fs.ReadStream} file - * @param {"assistants" | "fine-tune"} purpose + * @param {OpenAIClient.FileCreateParams['file']} file + * @param {OpenAIClient.FileCreateParams['purpose']} purpose + * @param {OpenAIClient.RequestOptions | undefined} options * @returns {Promise} */ - static async createFile({ file, purpose }: OpenAIFilesCreate) { - const oaiClient = new OpenAIClient(); - return oaiClient.files.create({ - file, - purpose, - }); + async createFile({ + file, + purpose, + options, + }: OpenAIClient.FileCreateParams & { + options?: OpenAIClient.RequestOptions; + }) { + return this.oaiClient.files.create({ file, purpose }, options); } /** @@ -34,11 +43,17 @@ export class OpenAIFiles extends Serializable { * @link {https://platform.openai.com/docs/api-reference/files/delete} * * @param {string} fileId + * @param {OpenAIClient.RequestOptions | undefined} options * @returns {Promise} */ - static async deleteFile({ fileId }: { fileId: string }) { - const oaiClient = new OpenAIClient(); - return oaiClient.files.del(fileId); + async deleteFile({ + fileId, + options, + }: { + fileId: string; + options?: OpenAIClient.RequestOptions; + }) { + return this.oaiClient.files.del(fileId, options); } /** @@ -49,12 +64,11 @@ export class OpenAIFiles extends Serializable { * @param {OpenAIClient.RequestOptions | undefined} options * @returns {Promise} */ - static async listFiles(props?: { + async listFiles(props?: { query?: OpenAIClient.Files.FileListParams; options?: OpenAIClient.RequestOptions; }) { - const oaiClient = new OpenAIClient(); - return oaiClient.files.list(props?.query, props?.options); + return this.oaiClient.files.list(props?.query, props?.options); } /** @@ -62,11 +76,17 @@ export class OpenAIFiles extends Serializable { * Returns information about a specific file. * @link {https://platform.openai.com/docs/api-reference/files/retrieve} * @param {string} fileId + * @param {OpenAIClient.RequestOptions | undefined} options * @returns {Promise} */ - static async retrieveFile({ fileId }: { fileId: string }) { - const oaiClient = new OpenAIClient(); - return oaiClient.files.retrieve(fileId); + async retrieveFile({ + fileId, + options, + }: { + fileId: string; + options?: OpenAIClient.RequestOptions; + }) { + return this.oaiClient.files.retrieve(fileId, options); } /** @@ -77,10 +97,16 @@ export class OpenAIFiles extends Serializable { * * @link {https://platform.openai.com/docs/api-reference/files/retrieve-contents} * @param {string} fileId + * @param {OpenAIClient.RequestOptions | undefined} options * @returns {Promise} */ - static async retrieveFileContent({ fileId }: { fileId: string }) { - const oaiClient = new OpenAIClient(); - return oaiClient.files.retrieveContent(fileId); + async retrieveFileContent({ + fileId, + options, + }: { + fileId: string; + options?: OpenAIClient.RequestOptions; + }) { + return this.oaiClient.files.retrieveContent(fileId, options); } } diff --git a/langchain/src/experimental/openai_files/tests/openai_file.int.test.ts b/langchain/src/experimental/openai_files/tests/openai_file.int.test.ts index 956bd80d4963..68d6ff128566 100644 --- a/langchain/src/experimental/openai_files/tests/openai_file.int.test.ts +++ b/langchain/src/experimental/openai_files/tests/openai_file.int.test.ts @@ -11,7 +11,8 @@ const __filename = fileURLToPath(import.meta.url); const __dirname = dirname(__filename); test("Use file with Open AI", async () => { - const file = await OpenAIFiles.createFile({ + const openAIFiles = new OpenAIFiles(); + const file = await openAIFiles.createFile({ file: fs.createReadStream(path.resolve(__dirname, `./test.jsonl`)), purpose: "fine-tune", }); @@ -28,7 +29,7 @@ test("Use file with Open AI", async () => { "purpose": "assistants", } */ - const fileContent = await OpenAIFiles.retrieveFileContent({ + const fileContent = await openAIFiles.retrieveFileContent({ fileId: file.id, }); console.log(fileContent); @@ -44,7 +45,7 @@ test("Use file with Open AI", async () => { "purpose": "assistants", } */ - const retrievedFile = await OpenAIFiles.retrieveFile({ + const retrievedFile = await openAIFiles.retrieveFile({ fileId: file.id, }); expect(retrievedFile.id).toBeDefined(); @@ -60,7 +61,7 @@ test("Use file with Open AI", async () => { "purpose": "assistants", } */ - const list = await OpenAIFiles.listFiles(); + const list = await openAIFiles.listFiles(); expect(list).toBeDefined(); expect(!!list.data.find((f) => f.id === file.id)).toBeTruthy(); /** @@ -74,7 +75,7 @@ test("Use file with Open AI", async () => { "purpose": "assistants", } */ - const result = await OpenAIFiles.deleteFile({ fileId: file.id }); + const result = await openAIFiles.deleteFile({ fileId: file.id }); expect(result.id).toBe(file.id); expect(result.deleted).toBeTruthy(); /** From cb213b3daedb835ec80a6008c63ae9c94fea92cc Mon Sep 17 00:00:00 2001 From: castroCrea Date: Wed, 15 Nov 2023 10:58:00 +0100 Subject: [PATCH 12/13] =?UTF-8?q?=F0=9F=93=9D=20clean=20the=20JSDoc?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- langchain/src/experimental/openai_files/index.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/langchain/src/experimental/openai_files/index.ts b/langchain/src/experimental/openai_files/index.ts index 2bd244a74cde..052778afecbc 100644 --- a/langchain/src/experimental/openai_files/index.ts +++ b/langchain/src/experimental/openai_files/index.ts @@ -20,7 +20,7 @@ export class OpenAIFiles extends Serializable { * Upload file * Upload a file that can be used across various endpoints. The size of all the files uploaded by one organization can be up to 100 GB. * - * The size of individual files can be a maximum of 512 MB. See the Assistants Tools guide to learn more about the types of files supported. The Fine-tuning API only supports .jsonl files. + * @note The size of individual files can be a maximum of 512 MB. See the Assistants Tools guide to learn more about the types of files supported. The Fine-tuning API only supports .jsonl files. * * @link {https://platform.openai.com/docs/api-reference/files/create} * @param {OpenAIClient.FileCreateParams['file']} file @@ -40,8 +40,8 @@ export class OpenAIFiles extends Serializable { /** * Delete a file. - * @link {https://platform.openai.com/docs/api-reference/files/delete} * + * @link {https://platform.openai.com/docs/api-reference/files/delete} * @param {string} fileId * @param {OpenAIClient.RequestOptions | undefined} options * @returns {Promise} @@ -59,6 +59,7 @@ export class OpenAIFiles extends Serializable { /** * List files * Returns a list of files that belong to the user's organization. + * * @link {https://platform.openai.com/docs/api-reference/files/list} * @param {OpenAIClient.Files.FileListParams | undefined} query * @param {OpenAIClient.RequestOptions | undefined} options @@ -74,6 +75,7 @@ export class OpenAIFiles extends Serializable { /** * Retrieve file * Returns information about a specific file. + * * @link {https://platform.openai.com/docs/api-reference/files/retrieve} * @param {string} fileId * @param {OpenAIClient.RequestOptions | undefined} options From c391073e9d95d8912a6b3b6c858cfbf2eeffff43 Mon Sep 17 00:00:00 2001 From: jacoblee93 Date: Mon, 20 Nov 2023 07:40:14 -0800 Subject: [PATCH 13/13] Use one documentation page --- .../agents/agent_types/openai_assistant.mdx | 162 +++++++++++++++++ .../agents/agent_types/openai_file.mdx | 166 ------------------ 2 files changed, 162 insertions(+), 166 deletions(-) delete mode 100644 docs/core_docs/docs/modules/agents/agent_types/openai_file.mdx diff --git a/docs/core_docs/docs/modules/agents/agent_types/openai_assistant.mdx b/docs/core_docs/docs/modules/agents/agent_types/openai_assistant.mdx index 6d62fd1105bd..3521f8432c1e 100644 --- a/docs/core_docs/docs/modules/agents/agent_types/openai_assistant.mdx +++ b/docs/core_docs/docs/modules/agents/agent_types/openai_assistant.mdx @@ -194,3 +194,165 @@ console.log(assistantResponse); ``` Here the assistant was able to utilize the `code_interpreter` tool to calculate the answer to our question. + +# OpenAI Files + +Files are used to upload documents that can be used with features like Assistants and Fine-tuning. + +We've implemented the File API in LangChain with create and delete. You can see [the official API reference here](https://platform.openai.com/docs/api-reference/files/object). + +The `File` object represents a document that has been uploaded to OpenAI. + +``` +{ + "id": "file-abc123", + "object": "file", + "bytes": 120000, + "created_at": 1677610602, + "filename": "salesOverview.pdf", + "purpose": "assistants", +} +``` + +## Create a File + +Upload a file that can be used across various endpoints. The size of all the files uploaded by one organization can be up to **100 GB**. + +The size of individual files can be a maximum of **512 MB**. See the Assistants Tools guide above to learn more about the types of files supported. The Fine-tuning API only supports `.jsonl` files. + +```typescript +import { OpenAIFiles } from "langchain/experimental/openai_files"; + +const openAIFiles = new OpenAIFiles(); +const file = await openAIFiles.create({ + file: fs.createReadStream(path.resolve(__dirname, `./test.txt`)), + purpose: "assistants", +}); +/** +* Output + { + "id": "file-BK7bzQj3FfZFXr7DbL6xJwfo", + "object": "file", + "bytes": 120000, + "created_at": 1677610602, + "filename": "salesOverview.pdf", + "purpose": "assistants", + } +*/ +``` + +## Use File in AI Assistant + +```typescript +import { OpenAIAssistantRunnable } from "langchain/experimental/openai_assistant"; +import { OpenAIFiles } from "langchain/experimental/openai_files"; + +const openAIFiles = new OpenAIFiles(); +const file = await openAIFiles.createFile({ + file: fs.createReadStream(path.resolve(__dirname, `./test.txt`)), + purpose: "assistants", +}); + +const agent = await OpenAIAssistantRunnable.createAssistant({ + model: "gpt-3.5-turbo-1106", + instructions: + "You are a weather bot. Use the provided functions to answer questions.", + name: "Weather Assistant", + tools, + asAgent: true, + file_ids: [file.id], +}); +``` + +## Delete a File + +Delete a file. + +```typescript +import { OpenAIFiles } from "langchain/experimental/openai_files"; + +const openAIFiles = new OpenAIFiles(); +const result = await openAIFiles.deleteFile({ fileId: file.id }); +/** +* Output: + { + "id": "file-abc123", + "object": "file", + "deleted": true + } +*/ +``` + +## List all Files + +Returns a list of files that belong to the user's organization. + +`purpose`?: string +Only return files with the given purpose. + +```typescript +import { OpenAIFiles } from "langchain/experimental/openai_files"; + +const openAIFiles = new OpenAIFiles(); +const result = await openAIFiles.listFiles({ purpose: "assistants" }); +/** +* Output: + { + "data": [ + { + "id": "file-abc123", + "object": "file", + "bytes": 175, + "created_at": 1613677385, + "filename": "salesOverview.pdf", + "purpose": "assistants", + }, + { + "id": "file-abc123", + "object": "file", + "bytes": 140, + "created_at": 1613779121, + "filename": "puppy.jsonl", + "purpose": "fine-tune", + } + ], + "object": "list" + } +*/ +``` + +## Retrieve File + +Returns information about a specific file. + +```typescript +import { OpenAIFiles } from "langchain/experimental/openai_files"; + +const openAIFiles = new OpenAIFiles(); +const result = await openAIFiles.retrieveFile({ fileId: file.id }); +/** +* Output: + { + "id": "file-abc123", + "object": "file", + "bytes": 120000, + "created_at": 1677610602, + "filename": "mydata.jsonl", + "purpose": "fine-tune", + } +*/ +``` + +## Retrieve File Content + +Returns the contents of the specified file. + +You can't retrieve the contents of a file that was uploaded with the "purpose": "assistants" API. + +```typescript +import { OpenAIFiles } from "langchain/experimental/openai_files"; + +const openAIFiles = new OpenAIFiles(); +const result = await openAIFiles.retrieveFileContent({ fileId: file.id }); +// Return the file content. +``` diff --git a/docs/core_docs/docs/modules/agents/agent_types/openai_file.mdx b/docs/core_docs/docs/modules/agents/agent_types/openai_file.mdx deleted file mode 100644 index 8b3903b80adc..000000000000 --- a/docs/core_docs/docs/modules/agents/agent_types/openai_file.mdx +++ /dev/null @@ -1,166 +0,0 @@ -# OpenAI File - -:::info -The [OpenAI File is used in OpenAI Assistant API](https://platform.openai.com/docs/assistants/how-it-works/creating-assistants) is still in beta. -[API reference](https://platform.openai.com/docs/api-reference/files/object) -::: - -Files are used to upload documents that can be used with features like Assistants and Fine-tuning. - -We've implemented the File API in LangChain with create and delete. - -The `File` object represents a document that has been uploaded to OpenAI. - -``` -{ - "id": "file-abc123", - "object": "file", - "bytes": 120000, - "created_at": 1677610602, - "filename": "salesOverview.pdf", - "purpose": "assistants", -} -``` - -## Create a File - -Upload a file that can be used across various endpoints. The size of all the files uploaded by one organization can be up to **100 GB**. - -The size of individual files can be a maximum of **512 MB**. See the Assistants Tools guide to learn more about the types of files supported. The Fine-tuning API only supports `.jsonl` files. - -```typescript -import { OpenAIFiles } from "langchain/experimental/openai_files"; - -const openAIFiles = new OpenAIFiles(); -const file = await openAIFiles.create({ - file: fs.createReadStream(path.resolve(__dirname, `./test.txt`)), - purpose: "assistants", -}); -/** -* Output - { - "id": "file-BK7bzQj3FfZFXr7DbL6xJwfo", - "object": "file", - "bytes": 120000, - "created_at": 1677610602, - "filename": "salesOverview.pdf", - "purpose": "assistants", - } -*/ -``` - -## Use File in AI Assistant - -```typescript -import { OpenAIAssistantRunnable } from "langchain/experimental/openai_assistant"; -import { OpenAIFiles } from "langchain/experimental/openai_files"; - -const openAIFiles = new OpenAIFiles(); -const file = await openAIFiles.createFile({ - file: fs.createReadStream(path.resolve(__dirname, `./test.txt`)), - purpose: "assistants", -}); - -const agent = await OpenAIAssistantRunnable.createAssistant({ - model: "gpt-3.5-turbo-1106", - instructions: - "You are a weather bot. Use the provided functions to answer questions.", - name: "Weather Assistant", - tools, - asAgent: true, - file_ids: [file.id], -}); -``` - -## Delete a File - -Delete a file. - -```typescript -import { OpenAIFiles } from "langchain/experimental/openai_files"; - -const openAIFiles = new OpenAIFiles(); -const result = await openAIFiles.deleteFile({ fileId: file.id }); -/** -* Output: - { - "id": "file-abc123", - "object": "file", - "deleted": true - } -*/ -``` - -## List all Files - -Returns a list of files that belong to the user's organization. - -`purpose`?: string -Only return files with the given purpose. - -```typescript -import { OpenAIFiles } from "langchain/experimental/openai_files"; - -const openAIFiles = new OpenAIFiles(); -const result = await openAIFiles.listFiles({ purpose: "assistants" }); -/** -* Output: - { - "data": [ - { - "id": "file-abc123", - "object": "file", - "bytes": 175, - "created_at": 1613677385, - "filename": "salesOverview.pdf", - "purpose": "assistants", - }, - { - "id": "file-abc123", - "object": "file", - "bytes": 140, - "created_at": 1613779121, - "filename": "puppy.jsonl", - "purpose": "fine-tune", - } - ], - "object": "list" - } -*/ -``` - -## Retrieve File - -Returns information about a specific file. - -```typescript -import { OpenAIFiles } from "langchain/experimental/openai_files"; - -const openAIFiles = new OpenAIFiles(); -const result = await openAIFiles.retrieveFile({ fileId: file.id }); -/** -* Output: - { - "id": "file-abc123", - "object": "file", - "bytes": 120000, - "created_at": 1677610602, - "filename": "mydata.jsonl", - "purpose": "fine-tune", - } -*/ -``` - -## Retrieve File Content - -Returns the contents of the specified file. - -You can't retrieve the contents of a file that was uploaded with the "purpose": "assistants" API. - -```typescript -import { OpenAIFiles } from "langchain/experimental/openai_files"; - -const openAIFiles = new OpenAIFiles(); -const result = await openAIFiles.retrieveFileContent({ fileId: file.id }); -// Return the file content. -```