From 8d8ed3f8d5b025d6b27629724346abd32bf13eee Mon Sep 17 00:00:00 2001 From: Vrv Date: Mon, 17 Jul 2023 10:31:38 +0530 Subject: [PATCH 1/2] Add Portkey for logging & monitoring requests --- .env.local.example | 5 ++++- README.md | 10 ++++++++++ src/app/api/qa-pg-vector/route.ts | 7 +++++-- src/app/api/qa-pinecone/route.ts | 8 ++++++-- src/scripts/PortkeyConfig.ts | 9 +++++++++ src/scripts/indexBlogPGVector.mjs | 3 ++- src/scripts/indexBlogs.mjs | 3 ++- 7 files changed, 38 insertions(+), 7 deletions(-) create mode 100644 src/scripts/PortkeyConfig.ts diff --git a/.env.local.example b/.env.local.example index 487e3f6f..6a914896 100644 --- a/.env.local.example +++ b/.env.local.example @@ -19,4 +19,7 @@ PINECONE_INDEX=ai**** # Supabase related environment variables SUPABASE_URL=https://**** -SUPABASE_PRIVATE_KEY=eyJ**** \ No newline at end of file +SUPABASE_PRIVATE_KEY=eyJ**** + +# Portkey related environment variables +PORTKEY_API_KEY= \ No newline at end of file diff --git a/README.md b/README.md index 36fc2a65..0672ce5d 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,7 @@ - App logic: [Next.js](https://nextjs.org/) - VectorDB: [Pinecone](https://www.pinecone.io/) / [Supabase pgvector](https://supabase.com/docs/guides/database/extensions/pgvector) - LLM Orchestration: [Langchain.js](https://js.langchain.com/docs/) +- Logs, Caching, Monitoring: [Portkey](https://docs.portkey.ai/) - Image Model: [Replicate](https://replicate.com/) - Text Model: [OpenAI](https://platform.openai.com/docs/models) - Text streaming: [ai sdk](https://github.com/vercel-labs/ai) @@ -76,6 +77,12 @@ e. **Supabase API key** - `SUPABASE_PRIVATE_KEY` is the key starts with `ey` under Project API Keys - Now, you should enable pgvector on Supabase and create a schema. You can do this easily by clicking on "SQL editor" on the left hand side on supabase UI and then clicking on "+New Query". Copy paste [this code snippet](https://github.com/a16z-infra/ai-getting-started/blob/main/pgvector.sql) in the SQL editor and click "Run". +f. **Portkey API key** +- Create a Portkey account [here](https://portkey.ai/). +- On the [dashboard](https://app.portkey.ai/), under the user settings on the top left, click on "Copy API key". +- Paste this to the 'PORTKEY_API_KEY' variable. +- Add Metadata, Caching, Retries and other headers [here](/src/scripts/PortkeyConfig.ts). Refer to [Portkey Docs](https://docs.portkey.ai/) for more. + ### 4. Generate embeddings There are a few markdown files under `/blogs` directory as examples so you can do Q&A on them. To generate embeddings and store them in the vector database for future queries, you can run the following command: @@ -112,6 +119,9 @@ Now you are ready to test out the app locally! To do this, simply run `npm run d - [Netlify](https://www.netlify.com/) - [Vercel](https://vercel.com/) +### 7. Log each request +- Now you should start seeing logs (cost, tokens, latency) for every request on [dashboard](https://app.portkey.ai/) +- Configure automatic retries, semantic caching, etc [here](/src/scripts/PortkeyConfig.ts) by referring to [Portkey Docs](https://docs.portkey.ai/). ## How to contribute to this repo diff --git a/src/app/api/qa-pg-vector/route.ts b/src/app/api/qa-pg-vector/route.ts index ce5dc568..57de725e 100644 --- a/src/app/api/qa-pg-vector/route.ts +++ b/src/app/api/qa-pg-vector/route.ts @@ -6,6 +6,7 @@ import dotenv from "dotenv"; import { VectorDBQAChain } from "langchain/chains"; import { StreamingTextResponse, LangChainStream } from "ai"; import { CallbackManager } from "langchain/callbacks"; +import { PortkeyConfig } from "../../../scripts/PortkeyConfig"; dotenv.config({ path: `.env.local` }); @@ -26,7 +27,7 @@ export async function POST(req: Request) { const client = createClient(url, privateKey, { auth }); const vectorStore = await SupabaseVectorStore.fromExistingIndex( - new OpenAIEmbeddings({ openAIApiKey: process.env.OPENAI_API_KEY }), + new OpenAIEmbeddings({ openAIApiKey: process.env.OPENAI_API_KEY }, PortkeyConfig), { client, tableName: "documents", @@ -41,7 +42,9 @@ export async function POST(req: Request) { modelName: "gpt-3.5-turbo-16k", openAIApiKey: process.env.OPENAI_API_KEY, callbackManager: CallbackManager.fromHandlers(handlers), - }); + }, + PortkeyConfig + ); const chain = VectorDBQAChain.fromLLM(model, vectorStore, { k: 1, diff --git a/src/app/api/qa-pinecone/route.ts b/src/app/api/qa-pinecone/route.ts index 4ef68338..0febb6c9 100644 --- a/src/app/api/qa-pinecone/route.ts +++ b/src/app/api/qa-pinecone/route.ts @@ -6,6 +6,8 @@ import { OpenAI } from "langchain/llms/openai"; import { PineconeStore } from "langchain/vectorstores/pinecone"; import { StreamingTextResponse, LangChainStream } from "ai"; import { CallbackManager } from "langchain/callbacks"; +import { PortkeyConfig } from "../../../scripts/PortkeyConfig"; + dotenv.config({ path: `.env.local` }); @@ -19,7 +21,7 @@ export async function POST(request: Request) { const pineconeIndex = client.Index(process.env.PINECONE_INDEX || ""); const vectorStore = await PineconeStore.fromExistingIndex( - new OpenAIEmbeddings({ openAIApiKey: process.env.OPENAI_API_KEY }), + new OpenAIEmbeddings({ openAIApiKey: process.env.OPENAI_API_KEY },PortkeyConfig), { pineconeIndex } ); @@ -29,7 +31,9 @@ export async function POST(request: Request) { modelName: "gpt-3.5-turbo-16k", openAIApiKey: process.env.OPENAI_API_KEY, callbackManager: CallbackManager.fromHandlers(handlers), - }); + }, + PortkeyConfig + ); const chain = VectorDBQAChain.fromLLM(model, vectorStore, { k: 1, diff --git a/src/scripts/PortkeyConfig.ts b/src/scripts/PortkeyConfig.ts new file mode 100644 index 00000000..8ac1ec7e --- /dev/null +++ b/src/scripts/PortkeyConfig.ts @@ -0,0 +1,9 @@ +export const PortkeyConfig = { + basePath: "https://api.portkey.ai/v1/proxy", + baseOptions: { + headers: { + "x-portkey-api-key": process.env.PORTKEY_API_KEY, + "x-portkey-mode": "proxy openai", + }, + } +} \ No newline at end of file diff --git a/src/scripts/indexBlogPGVector.mjs b/src/scripts/indexBlogPGVector.mjs index 640305a8..6a2fb168 100644 --- a/src/scripts/indexBlogPGVector.mjs +++ b/src/scripts/indexBlogPGVector.mjs @@ -7,6 +7,7 @@ import { OpenAIEmbeddings } from "langchain/embeddings/openai"; import { SupabaseVectorStore } from "langchain/vectorstores/supabase"; import { createClient } from "@supabase/supabase-js"; import { RecursiveCharacterTextSplitter } from "langchain/text_splitter"; +import { PortkeyConfig } from "./PortkeyConfig"; import fs from "fs"; import path from "path"; @@ -47,7 +48,7 @@ const client = createClient( await SupabaseVectorStore.fromDocuments( langchainDocs.flat(), - new OpenAIEmbeddings({ openAIApiKey: process.env.OPENAI_API_KEY }), + new OpenAIEmbeddings({ openAIApiKey: process.env.OPENAI_API_KEY },PortkeyConfig), { client, tableName: "documents", diff --git a/src/scripts/indexBlogs.mjs b/src/scripts/indexBlogs.mjs index 074a9765..e92a51aa 100644 --- a/src/scripts/indexBlogs.mjs +++ b/src/scripts/indexBlogs.mjs @@ -6,6 +6,7 @@ import { OpenAIEmbeddings } from "langchain/embeddings/openai"; import { PineconeStore } from "langchain/vectorstores/pinecone"; import fs from "fs"; import path from "path"; +import { PortkeyConfig } from "./PortkeyConfig"; dotenv.config({ path: `.env.local` }); @@ -28,7 +29,7 @@ const pineconeIndex = client.Index(process.env.PINECONE_INDEX); await PineconeStore.fromDocuments( lanchainDocs, - new OpenAIEmbeddings({ openAIApiKey: process.env.OPENAI_API_KEY }), + new OpenAIEmbeddings({ openAIApiKey: process.env.OPENAI_API_KEY },PortkeyConfig), { pineconeIndex, } From b7b618fe4abd2afc1a2d8a56ebca5b991e72b219 Mon Sep 17 00:00:00 2001 From: Vrv Date: Mon, 17 Jul 2023 14:26:46 +0530 Subject: [PATCH 2/2] Add Portkey config (.env fixes) --- src/app/api/qa-pg-vector/route.ts | 2 +- src/app/api/qa-pinecone/route.ts | 2 +- src/scripts/{PortkeyConfig.ts => PortkeyConfig.mjs} | 5 +++++ src/scripts/indexBlogPGVector.mjs | 2 +- src/scripts/indexBlogs.mjs | 2 +- 5 files changed, 9 insertions(+), 4 deletions(-) rename src/scripts/{PortkeyConfig.ts => PortkeyConfig.mjs} (69%) diff --git a/src/app/api/qa-pg-vector/route.ts b/src/app/api/qa-pg-vector/route.ts index 57de725e..1b06abee 100644 --- a/src/app/api/qa-pg-vector/route.ts +++ b/src/app/api/qa-pg-vector/route.ts @@ -6,7 +6,7 @@ import dotenv from "dotenv"; import { VectorDBQAChain } from "langchain/chains"; import { StreamingTextResponse, LangChainStream } from "ai"; import { CallbackManager } from "langchain/callbacks"; -import { PortkeyConfig } from "../../../scripts/PortkeyConfig"; +import { PortkeyConfig } from "../../../scripts/PortkeyConfig.mjs"; dotenv.config({ path: `.env.local` }); diff --git a/src/app/api/qa-pinecone/route.ts b/src/app/api/qa-pinecone/route.ts index 0febb6c9..ec20a65d 100644 --- a/src/app/api/qa-pinecone/route.ts +++ b/src/app/api/qa-pinecone/route.ts @@ -6,7 +6,7 @@ import { OpenAI } from "langchain/llms/openai"; import { PineconeStore } from "langchain/vectorstores/pinecone"; import { StreamingTextResponse, LangChainStream } from "ai"; import { CallbackManager } from "langchain/callbacks"; -import { PortkeyConfig } from "../../../scripts/PortkeyConfig"; +import { PortkeyConfig } from "../../../scripts/PortkeyConfig.mjs"; dotenv.config({ path: `.env.local` }); diff --git a/src/scripts/PortkeyConfig.ts b/src/scripts/PortkeyConfig.mjs similarity index 69% rename from src/scripts/PortkeyConfig.ts rename to src/scripts/PortkeyConfig.mjs index 8ac1ec7e..98685e4c 100644 --- a/src/scripts/PortkeyConfig.ts +++ b/src/scripts/PortkeyConfig.mjs @@ -1,3 +1,8 @@ +import dotenv from "dotenv"; +import path from "path"; + +dotenv.config({ path: `.env.local` }); + export const PortkeyConfig = { basePath: "https://api.portkey.ai/v1/proxy", baseOptions: { diff --git a/src/scripts/indexBlogPGVector.mjs b/src/scripts/indexBlogPGVector.mjs index 6a2fb168..155483db 100644 --- a/src/scripts/indexBlogPGVector.mjs +++ b/src/scripts/indexBlogPGVector.mjs @@ -7,7 +7,7 @@ import { OpenAIEmbeddings } from "langchain/embeddings/openai"; import { SupabaseVectorStore } from "langchain/vectorstores/supabase"; import { createClient } from "@supabase/supabase-js"; import { RecursiveCharacterTextSplitter } from "langchain/text_splitter"; -import { PortkeyConfig } from "./PortkeyConfig"; +import { PortkeyConfig } from "./PortkeyConfig.mjs"; import fs from "fs"; import path from "path"; diff --git a/src/scripts/indexBlogs.mjs b/src/scripts/indexBlogs.mjs index e92a51aa..274a1950 100644 --- a/src/scripts/indexBlogs.mjs +++ b/src/scripts/indexBlogs.mjs @@ -6,7 +6,7 @@ import { OpenAIEmbeddings } from "langchain/embeddings/openai"; import { PineconeStore } from "langchain/vectorstores/pinecone"; import fs from "fs"; import path from "path"; -import { PortkeyConfig } from "./PortkeyConfig"; +import { PortkeyConfig } from "./PortkeyConfig.mjs"; dotenv.config({ path: `.env.local` });