Merge pull request ai16z#194 from juke/Groq-API-Integration

Groq api integration
ProphetX10 · Nov 4, 2024 · aca0cdd · aca0cdd
2 parents 2e440a0 + 96236b2
commit aca0cdd
Show file tree

Hide file tree

Showing 4 changed files with 39 additions and 0 deletions.
diff --git a/core/.env.example b/core/.env.example
@@ -2,6 +2,8 @@
 DISCORD_APPLICATION_ID=
 DISCORD_API_TOKEN= # Bot token
 OPENAI_API_KEY=sk-* # OpenAI API key, starting with sk-
+GROQ_API_KEY=gsk_*
+
 ELEVENLABS_XI_API_KEY= # API key from elevenlabs
 
 # ELEVENLABS SETTINGS

diff --git a/core/src/core/generation.ts b/core/src/core/generation.ts
@@ -131,6 +131,24 @@ export async function generateText({
                 break;
             }
 
+            case ModelProvider.GROQ: {
+                console.log("Initializing Groq model.");
+                const groq = createGroq({ apiKey });
+
+                const { text: groqResponse } = await aiGenerateText({
+                    model: groq.languageModel(model),
+                    prompt: context,
+                    temperature: temperature,
+                    maxTokens: max_response_length,
+                    frequencyPenalty: frequency_penalty,
+                    presencePenalty: presence_penalty,
+                });
+
+                response = groqResponse;
+                console.log("Received response from Groq model.");
+                break;
+            }
+
             case ModelProvider.LLAMALOCAL:
                 prettyConsole.log(
                     "Using local Llama model for text completion."

diff --git a/core/src/core/models.ts b/core/src/core/models.ts
@@ -4,6 +4,7 @@ type Models = {
     [ModelProvider.OPENAI]: Model;
     [ModelProvider.ANTHROPIC]: Model;
     [ModelProvider.GROK]: Model;
+    [ModelProvider.GROQ]: Model;
     [ModelProvider.LLAMACLOUD]: Model;
     [ModelProvider.LLAMALOCAL]: Model;
     [ModelProvider.GOOGLE]: Model;
@@ -78,6 +79,23 @@ const models: Models = {
             [ModelClass.EMBEDDING]: "grok-2-beta", // not sure about this one
         },
     },
+    [ModelProvider.GROQ]: {
+        endpoint: "https://api.groq.com/openai/v1",
+        settings: {
+            stop: [],
+            maxInputTokens: 128000,
+            maxOutputTokens: 8000,
+            frequency_penalty: 0.0,
+            presence_penalty: 0.0,
+            temperature: 0.3,
+        },
+        model: {
+            [ModelClass.SMALL]: "llama-3.1-8b-instant",
+            [ModelClass.MEDIUM]: "llama-3.1-70b-versatile",
+            [ModelClass.LARGE]: "llama-3.2-90b-text-preview",
+            [ModelClass.EMBEDDING]: "llama-3.1-8b-instant",
+        },
+    },
     [ModelProvider.LLAMACLOUD]: {
         settings: {
             stop: [],

diff --git a/core/src/core/types.ts b/core/src/core/types.ts
@@ -102,6 +102,7 @@ export enum ModelProvider {
     OPENAI = "openai",
     ANTHROPIC = "anthropic",
     GROK = "grok",
+    GROQ = "groq",
     LLAMACLOUD = "llama_cloud",
     LLAMALOCAL = "llama_local",
     GOOGLE = "google",