From d61dccc162485897969b65a419553b13d54dda10 Mon Sep 17 00:00:00 2001 From: Tsukasa OISHI Date: Sat, 2 Dec 2023 21:21:58 +0900 Subject: [PATCH 1/5] Remove unused option --- langchain-core/src/utils/tiktoken.ts | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/langchain-core/src/utils/tiktoken.ts b/langchain-core/src/utils/tiktoken.ts index a823b5b18637..2545f7d56dd8 100644 --- a/langchain-core/src/utils/tiktoken.ts +++ b/langchain-core/src/utils/tiktoken.ts @@ -13,16 +13,10 @@ const caller = /* #__PURE__ */ new AsyncCaller({}); export async function getEncoding( encoding: TiktokenEncoding, - options?: { - signal?: AbortSignal; - extendedSpecialTokens?: Record; - } ) { if (!(encoding in cache)) { cache[encoding] = caller - .fetch(`https://tiktoken.pages.dev/js/${encoding}.json`, { - signal: options?.signal, - }) + .fetch(`https://tiktoken.pages.dev/js/${encoding}.json`) .then((res) => res.json()) .catch((e) => { delete cache[encoding]; @@ -30,15 +24,11 @@ export async function getEncoding( }); } - return new Tiktoken(await cache[encoding], options?.extendedSpecialTokens); + return new Tiktoken(await cache[encoding]); } export async function encodingForModel( model: TiktokenModel, - options?: { - signal?: AbortSignal; - extendedSpecialTokens?: Record; - } ) { - return getEncoding(getEncodingNameForModel(model), options); + return getEncoding(getEncodingNameForModel(model)); } From 306cd865e19f95f9ef4a14bf967169706b2e3756 Mon Sep 17 00:00:00 2001 From: Tsukasa OISHI Date: Mon, 4 Dec 2023 09:46:14 +0900 Subject: [PATCH 2/5] Cache the Tiktoken object --- langchain-core/src/utils/tiktoken.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/langchain-core/src/utils/tiktoken.ts b/langchain-core/src/utils/tiktoken.ts index 2545f7d56dd8..2c2838c438c0 100644 --- a/langchain-core/src/utils/tiktoken.ts +++ b/langchain-core/src/utils/tiktoken.ts @@ -1,13 +1,12 @@ import { Tiktoken, - TiktokenBPE, TiktokenEncoding, TiktokenModel, getEncodingNameForModel, } from "js-tiktoken/lite"; import { AsyncCaller } from "./async_caller.js"; -const cache: Record> = {}; +const cache: Record> = {}; const caller = /* #__PURE__ */ new AsyncCaller({}); @@ -18,13 +17,14 @@ export async function getEncoding( cache[encoding] = caller .fetch(`https://tiktoken.pages.dev/js/${encoding}.json`) .then((res) => res.json()) + .then((data) => new Tiktoken(data)) .catch((e) => { delete cache[encoding]; throw e; }); } - return new Tiktoken(await cache[encoding]); + return await cache[encoding]; } export async function encodingForModel( From 09d1c461c14e595d1baeb04d22f69865eaf70dbc Mon Sep 17 00:00:00 2001 From: jacoblee93 Date: Mon, 4 Dec 2023 14:27:22 -0800 Subject: [PATCH 3/5] Fix format --- langchain-core/src/utils/tiktoken.ts | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/langchain-core/src/utils/tiktoken.ts b/langchain-core/src/utils/tiktoken.ts index 2c2838c438c0..be930b41eb9d 100644 --- a/langchain-core/src/utils/tiktoken.ts +++ b/langchain-core/src/utils/tiktoken.ts @@ -10,9 +10,7 @@ const cache: Record> = {}; const caller = /* #__PURE__ */ new AsyncCaller({}); -export async function getEncoding( - encoding: TiktokenEncoding, -) { +export async function getEncoding(encoding: TiktokenEncoding) { if (!(encoding in cache)) { cache[encoding] = caller .fetch(`https://tiktoken.pages.dev/js/${encoding}.json`) @@ -27,8 +25,6 @@ export async function getEncoding( return await cache[encoding]; } -export async function encodingForModel( - model: TiktokenModel, -) { +export async function encodingForModel(model: TiktokenModel) { return getEncoding(getEncodingNameForModel(model)); } From 7cb6100b89cd1e3303e9db1b8d4a7bd95695eac5 Mon Sep 17 00:00:00 2001 From: jacoblee93 Date: Mon, 4 Dec 2023 14:30:45 -0800 Subject: [PATCH 4/5] Bump core version --- langchain-core/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/langchain-core/package.json b/langchain-core/package.json index 1927a865c10c..33b321bc784f 100644 --- a/langchain-core/package.json +++ b/langchain-core/package.json @@ -1,6 +1,6 @@ { "name": "@langchain/core", - "version": "0.0.7", + "version": "0.0.8", "description": "Core LangChain.js abstractions and schemas", "type": "module", "engines": { From 6671fd451ee3ae6c242c53a66e0ea6a48e31ead4 Mon Sep 17 00:00:00 2001 From: Tat Dat Duong Date: Mon, 4 Dec 2023 23:40:27 +0100 Subject: [PATCH 5/5] Upgrade to js-tiktoken@1.0.8 --- langchain-core/package.json | 2 +- yarn.lock | 11 ++++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/langchain-core/package.json b/langchain-core/package.json index 33b321bc784f..a11c036e3a4d 100644 --- a/langchain-core/package.json +++ b/langchain-core/package.json @@ -37,7 +37,7 @@ "ansi-styles": "^5.0.0", "camelcase": "6", "decamelize": "1.2.0", - "js-tiktoken": "^1.0.7", + "js-tiktoken": "^1.0.8", "langsmith": "^0.0.48", "p-queue": "^6.6.2", "p-retry": "4", diff --git a/yarn.lock b/yarn.lock index ae44b3e678f9..d1f3163cabf2 100644 --- a/yarn.lock +++ b/yarn.lock @@ -8038,7 +8038,7 @@ __metadata: eslint-plugin-prettier: ^4.2.1 jest: ^29.5.0 jest-environment-node: ^29.6.4 - js-tiktoken: ^1.0.7 + js-tiktoken: ^1.0.8 langsmith: ^0.0.48 p-queue: ^6.6.2 p-retry: 4 @@ -22255,6 +22255,15 @@ __metadata: languageName: node linkType: hard +"js-tiktoken@npm:^1.0.8": + version: 1.0.8 + resolution: "js-tiktoken@npm:1.0.8" + dependencies: + base64-js: ^1.5.1 + checksum: ac6e666f14661b4e744bd60987e35275668863a686413bb35baca2d9a503bc95fcfa907f3c02b8e8abe4a4a70abe622c06cbb72ce3574b3b929d22f9146c3f85 + languageName: node + linkType: hard + "js-tokens@npm:^3.0.0 || ^4.0.0, js-tokens@npm:^4.0.0": version: 4.0.0 resolution: "js-tokens@npm:4.0.0"