Add Portkey for logging, monitoring requests

.env.local.example

@@ -19,4 +19,7 @@ PINECONE_INDEX=ai****
 
 # Supabase related environment variables
 SUPABASE_URL=https://****
-SUPABASE_PRIVATE_KEY=eyJ****
+SUPABASE_PRIVATE_KEY=eyJ****
+
+# Portkey related environment variables
+PORTKEY_API_KEY=

README.md

@@ -15,6 +15,7 @@
 - App logic: [Next.js](https://nextjs.org/)
 - VectorDB: [Pinecone](https://www.pinecone.io/) / [Supabase pgvector](https://supabase.com/docs/guides/database/extensions/pgvector)
 - LLM Orchestration: [Langchain.js](https://js.langchain.com/docs/) 
+- Logs, Caching, Monitoring: [Portkey](https://docs.portkey.ai/) 
 - Image Model: [Replicate](https://replicate.com/)
 - Text Model: [OpenAI](https://platform.openai.com/docs/models)
 - Text streaming: [ai sdk](https://github.com/vercel-labs/ai)
@@ -76,6 +77,12 @@ e. **Supabase API key**
 - `SUPABASE_PRIVATE_KEY` is the key starts with `ey` under Project API Keys
 - Now, you should enable pgvector on Supabase and create a schema. You can do this easily by clicking on "SQL editor" on the left hand side on supabase UI and then clicking on "+New Query". Copy paste [this code snippet](https://github.com/a16z-infra/ai-getting-started/blob/main/pgvector.sql) in the SQL editor and click "Run".
 
+f. **Portkey API key**
+- Create a Portkey account [here](https://portkey.ai/).
+- On the [dashboard](https://app.portkey.ai/), under the user settings on the top left, click on "Copy API key".
+- Paste this to the 'PORTKEY_API_KEY' variable.
+- Add Metadata, Caching, Retries and other headers [here](/src/scripts/PortkeyConfig.ts). Refer to [Portkey Docs](https://docs.portkey.ai/) for more.
+
 ### 4. Generate embeddings 
 
 There are a few markdown files under `/blogs` directory as examples so you can do Q&A on them. To generate embeddings and store them in the vector database for future queries, you can run the following command: 
@@ -112,6 +119,9 @@ Now you are ready to test out the app locally! To do this, simply run `npm run d
 - [Netlify](https://www.netlify.com/)
 - [Vercel](https://vercel.com/)
 
+### 7. Log each request
+- Now you should start seeing logs (cost, tokens, latency) for every request on [dashboard](https://app.portkey.ai/)
+- Configure automatic retries, semantic caching, etc [here](/src/scripts/PortkeyConfig.ts) by referring to [Portkey Docs](https://docs.portkey.ai/).
 
 ## How to contribute to this repo
 

src/app/api/qa-pg-vector/route.ts

@@ -6,6 +6,7 @@ import dotenv from "dotenv";
 import { VectorDBQAChain } from "langchain/chains";
 import { StreamingTextResponse, LangChainStream } from "ai";
 import { CallbackManager } from "langchain/callbacks";
+import { PortkeyConfig } from "../../../scripts/PortkeyConfig.mjs";
 
 dotenv.config({ path: `.env.local` });
 
@@ -26,7 +27,7 @@ export async function POST(req: Request) {
   const client = createClient(url, privateKey, { auth });
 
   const vectorStore = await SupabaseVectorStore.fromExistingIndex(
-    new OpenAIEmbeddings({ openAIApiKey: process.env.OPENAI_API_KEY }),
+    new OpenAIEmbeddings({ openAIApiKey: process.env.OPENAI_API_KEY }, PortkeyConfig),
     {
       client,
       tableName: "documents",
@@ -41,7 +42,9 @@ export async function POST(req: Request) {
     modelName: "gpt-3.5-turbo-16k",
     openAIApiKey: process.env.OPENAI_API_KEY,
     callbackManager: CallbackManager.fromHandlers(handlers),
-  });
+    },
+    PortkeyConfig
+  );
 
   const chain = VectorDBQAChain.fromLLM(model, vectorStore, {
     k: 1,

src/app/api/qa-pinecone/route.ts

@@ -6,6 +6,8 @@ import { OpenAI } from "langchain/llms/openai";
 import { PineconeStore } from "langchain/vectorstores/pinecone";
 import { StreamingTextResponse, LangChainStream } from "ai";
 import { CallbackManager } from "langchain/callbacks";
+import { PortkeyConfig } from "../../../scripts/PortkeyConfig.mjs";
+
 
 dotenv.config({ path: `.env.local` });
 
@@ -19,7 +21,7 @@ export async function POST(request: Request) {
   const pineconeIndex = client.Index(process.env.PINECONE_INDEX || "");
 
   const vectorStore = await PineconeStore.fromExistingIndex(
-    new OpenAIEmbeddings({ openAIApiKey: process.env.OPENAI_API_KEY }),
+    new OpenAIEmbeddings({ openAIApiKey: process.env.OPENAI_API_KEY },PortkeyConfig),
     { pineconeIndex }
   );
 
@@ -29,7 +31,9 @@ export async function POST(request: Request) {
     modelName: "gpt-3.5-turbo-16k",
     openAIApiKey: process.env.OPENAI_API_KEY,
     callbackManager: CallbackManager.fromHandlers(handlers),
-  });
+    },
+    PortkeyConfig
+  );
 
   const chain = VectorDBQAChain.fromLLM(model, vectorStore, {
     k: 1,

src/scripts/PortkeyConfig.mjs

@@ -0,0 +1,14 @@
+import dotenv from "dotenv";
+import path from "path";
+
+dotenv.config({ path: `.env.local` });
+
+export const PortkeyConfig = {
+  basePath: "https://api.portkey.ai/v1/proxy",
+  baseOptions: {
+    headers: {
+      "x-portkey-api-key": process.env.PORTKEY_API_KEY,
+      "x-portkey-mode": "proxy openai",
+    },
+  }
+}

src/scripts/indexBlogPGVector.mjs

@@ -7,6 +7,7 @@ import { OpenAIEmbeddings } from "langchain/embeddings/openai";
 import { SupabaseVectorStore } from "langchain/vectorstores/supabase";
 import { createClient } from "@supabase/supabase-js";
 import { RecursiveCharacterTextSplitter } from "langchain/text_splitter";
+import { PortkeyConfig } from "./PortkeyConfig.mjs";
 
 import fs from "fs";
 import path from "path";
@@ -47,7 +48,7 @@ const client = createClient(
 
 await SupabaseVectorStore.fromDocuments(
   langchainDocs.flat(),
-  new OpenAIEmbeddings({ openAIApiKey: process.env.OPENAI_API_KEY }),
+  new OpenAIEmbeddings({ openAIApiKey: process.env.OPENAI_API_KEY },PortkeyConfig),
   {
     client,
     tableName: "documents",

src/scripts/indexBlogs.mjs

@@ -6,6 +6,7 @@ import { OpenAIEmbeddings } from "langchain/embeddings/openai";
 import { PineconeStore } from "langchain/vectorstores/pinecone";
 import fs from "fs";
 import path from "path";
+import { PortkeyConfig } from "./PortkeyConfig.mjs";
 
 dotenv.config({ path: `.env.local` });
 
@@ -28,7 +29,7 @@ const pineconeIndex = client.Index(process.env.PINECONE_INDEX);
 
 await PineconeStore.fromDocuments(
   lanchainDocs,
-  new OpenAIEmbeddings({ openAIApiKey: process.env.OPENAI_API_KEY }),
+  new OpenAIEmbeddings({ openAIApiKey: process.env.OPENAI_API_KEY },PortkeyConfig),
   {
     pineconeIndex,
   }