🎨 refactoring for continue-proxy LLM (#1277)

core/continueServer/interface.ts

@@ -19,6 +19,9 @@ export interface EmbeddingsCacheResponse<T extends ArtifactType> {
 }
 
 export interface IContinueServerClient {
+  connected: boolean;
+  url: URL | undefined;
+  getUserToken(): Promise<string | undefined>;
   getConfig(): Promise<{ configJson: string; configJs: string }>;
   getFromIndexCache<T extends ArtifactType>(
     keys: string[],

core/continueServer/stubs/client.ts

@@ -5,10 +5,28 @@ import type {
 } from "../interface.js";
 
 export class ContinueServerClient implements IContinueServerClient {
+  url: URL | undefined;
+
   constructor(
-    private readonly serverUrl: string,
+    serverUrl: string | undefined,
     private readonly userToken: Promise<string | undefined>,
-  ) {}
+  ) {
+    try {
+      this.url =
+        typeof serverUrl !== "string" || serverUrl === ""
+          ? undefined
+          : new URL(serverUrl);
+    } catch (e) {
+      console.warn("Invalid Continue server url", e);
+      this.url = undefined;
+    }
+  }
+
+  getUserToken(): Promise<string | undefined> {
+    return this.userToken;
+  }
+
+  connected: boolean = false;
 
   public async getConfig(): Promise<{ configJson: string; configJs: string }> {
     throw new Error("Not Implemented");

core/index.d.ts

@@ -545,6 +545,7 @@ type ModelProvider =
   | "deepinfra"
   | "flowise"
   | "groq"
+  | "continue-proxy"
   | "custom";
 
 export type ModelName =

core/indexing/LanceDbIndex.ts

@@ -1,14 +1,14 @@
 // NOTE: vectordb requirement must be listed in extensions/vscode to avoid error
 import { v4 as uuidv4 } from "uuid";
-import type { Table } from "vectordb";
-import type {
+import { Table } from "vectordb";
+import { IContinueServerClient } from "../continueServer/interface.js";
+import {
   BranchAndDir,
   Chunk,
   EmbeddingsProvider,
   IndexTag,
   IndexingProgressUpdate,
 } from "../index.js";
-import { ContinueServerClient } from "../continueServer/stubs/client.js";
 import { MAX_CHUNK_SIZE } from "../llm/constants.js";
 import { getBasename } from "../util/index.js";
 import { getLanceDbPath } from "../util/paths.js";
@@ -39,7 +39,7 @@ export class LanceDbIndex implements CodebaseIndex {
   constructor(
     private readonly embeddingsProvider: EmbeddingsProvider,
     private readonly readFile: (filepath: string) => Promise<string>,
-    private readonly continueServerClient?: ContinueServerClient,
+    private readonly continueServerClient: IContinueServerClient,
   ) {}
 
   private tableNameForTag(tag: IndexTag) {
@@ -175,7 +175,7 @@ export class LanceDbIndex implements CodebaseIndex {
     };
 
     // Check remote cache
-    if (this.continueServerClient !== undefined) {
+    if (this.continueServerClient.connected) {
       try {
         const keys = results.compute.map(({ cacheKey }) => cacheKey);
         const resp = await this.continueServerClient.getFromIndexCache(

core/indexing/chunk/ChunkCodebaseIndex.ts

@@ -1,5 +1,5 @@
+import { IContinueServerClient } from "../../continueServer/interface.js";
 import { Chunk, IndexTag, IndexingProgressUpdate } from "../../index.js";
-import { ContinueServerClient } from "../../continueServer/stubs/client.js";
 import { MAX_CHUNK_SIZE } from "../../llm/constants.js";
 import { getBasename } from "../../util/index.js";
 import { DatabaseConnection, SqliteDb, tagToString } from "../refreshIndex.js";
@@ -22,7 +22,7 @@ export class ChunkCodebaseIndex implements CodebaseIndex {
 
   constructor(
     private readonly readFile: (filepath: string) => Promise<string>,
-    private readonly continueServerClient?: ContinueServerClient,
+    private readonly continueServerClient: IContinueServerClient,
   ) {
     this.readFile = readFile;
   }
@@ -76,7 +76,7 @@ export class ChunkCodebaseIndex implements CodebaseIndex {
     }
 
     // Check the remote cache
-    if (this.continueServerClient !== undefined) {
+    if (this.continueServerClient.connected) {
       try {
         const keys = results.compute.map(({ cacheKey }) => cacheKey);
         const resp = await this.continueServerClient.getFromIndexCache(

core/indexing/indexCodebase.ts

@@ -1,6 +1,6 @@
-import { IDE, IndexTag, IndexingProgressUpdate } from "../index.js";
 import { ConfigHandler } from "../config/handler.js";
-import { ContinueServerClient } from "../continueServer/stubs/client.js";
+import { IContinueServerClient } from "../continueServer/interface.js";
+import { IDE, IndexTag, IndexingProgressUpdate } from "../index.js";
 import { CodeSnippetsCodebaseIndex } from "./CodeSnippetsIndex.js";
 import { FullTextSearchCodebaseIndex } from "./FullTextSearch.js";
 import { LanceDbIndex } from "./LanceDbIndex.js";
@@ -21,21 +21,12 @@ export class PauseToken {
 }
 
 export class CodebaseIndexer {
-  private continueServerClient?: ContinueServerClient;
   constructor(
     private readonly configHandler: ConfigHandler,
     private readonly ide: IDE,
     private readonly pauseToken: PauseToken,
-    private readonly continueServerUrl: string | undefined,
-    private readonly userToken: string | undefined,
-  ) {
-    if (this.continueServerUrl) {
-      this.continueServerClient = new ContinueServerClient(
-        this.continueServerUrl,
-        Promise.resolve(this.userToken),
-      );
-    }
-  }
+    private readonly continueServerClient: IContinueServerClient,
+  ) {}
 
   private async getIndexesToBuild(): Promise<CodebaseIndex[]> {
     const config = await this.configHandler.loadConfig();

core/llm/autodetect.ts

@@ -41,6 +41,7 @@ const PROVIDER_HANDLES_TEMPLATING: ModelProvider[] = [
   "together",
   "anthropic",
   "bedrock",
+  "continue-proxy",
 ];
 
 const PROVIDER_SUPPORTS_IMAGES: ModelProvider[] = [
@@ -50,6 +51,7 @@ const PROVIDER_SUPPORTS_IMAGES: ModelProvider[] = [
   "free-trial",
   "anthropic",
   "bedrock",
+  "continue-proxy",
 ];
 
 const MODEL_SUPPORTS_IMAGES: string[] = [

core/llm/llms/index.ts

@@ -27,6 +27,7 @@ import OpenAIFreeTrial from "./OpenAIFreeTrial.js";
 import Replicate from "./Replicate.js";
 import TextGenWebUI from "./TextGenWebUI.js";
 import Together from "./Together.js";
+import ContinueProxy from "./stubs/ContinueProxy.js";
 
 function convertToLetter(num: number): string {
 	let result = "";
@@ -76,26 +77,27 @@ export async function renderTemplatedString(
 }
 
 const LLMs = [
-	Anthropic,
-	Cohere,
-	FreeTrial,
-	Gemini,
-	Llamafile,
-	Ollama,
-	Replicate,
-	TextGenWebUI,
-	Together,
-	HuggingFaceTGI,
-	HuggingFaceInferenceAPI,
-	LlamaCpp,
-	OpenAI,
-	LMStudio,
-	Mistral,
-	Bedrock,
-	DeepInfra,
-	OpenAIFreeTrial,
-	Flowise,
-	Groq,
+  Anthropic,
+  Cohere,
+  FreeTrial,
+  Gemini,
+  Llamafile,
+  Ollama,
+  Replicate,
+  TextGenWebUI,
+  Together,
+  HuggingFaceTGI,
+  HuggingFaceInferenceAPI,
+  LlamaCpp,
+  OpenAI,
+  LMStudio,
+  Mistral,
+  Bedrock,
+  DeepInfra,
+  OpenAIFreeTrial,
+  Flowise,
+  Groq,
+  ContinueProxy,
 ];
 
 export async function llmFromDescription(
@@ -134,7 +136,7 @@ export async function llmFromDescription(
     },
     systemMessage,
     writeLog,
-    uniqueId
+    uniqueId,
   };
 
 	return new cls(options);

core/llm/llms/stubs/ContinueProxy.ts

@@ -1,11 +1,8 @@
-import { LLMOptions, ModelProvider } from "../../..";
+import { ModelProvider } from "../../..";
 import OpenAI from "../OpenAI";
 
 class ContinueProxy extends OpenAI {
   static providerName: ModelProvider = "continue-proxy";
-  static defaultOptions: Partial<LLMOptions> = {
-    apiBase: "http://localhost:3000/proxy/v1",
-  };
 }
 
 export default ContinueProxy;

docs/static/schemas/config.json

@@ -156,7 +156,8 @@
             "mistral",
             "deepinfra",
             "flowise",
-            "groq"
+            "groq",
+            "continue-proxy"
           ],
           "markdownEnumDescriptions": [
             "### OpenAI\nUse gpt-4, gpt-3.5-turbo, or any other OpenAI model. See [here](https://openai.com/product#made-for-developers) to obtain an API key.\n\n> [Reference](https://docs.continue.dev/reference/Model%20Providers/openai)",
@@ -174,7 +175,8 @@
             "### LMStudio\nLMStudio provides a professional and well-designed GUI for exploring, configuring, and serving LLMs. It is available on both Mac and Windows. To get started:\n1. Download from [lmstudio.ai](https://lmstudio.ai/) and open the application\n2. Search for and download the desired model from the home screen of LMStudio.\n3. In the left-bar, click the '<->' icon to open the Local Inference Server and press 'Start Server'.\n4. Once your model is loaded and the server has started, you can begin using Continue.\n> [Reference](https://continue.dev/docs/reference/Model%20Providers/lmstudio)",
             "### Llamafile\nTo get started with llamafiles, find and download a binary on their [GitHub repo](https://github.com/Mozilla-Ocho/llamafile#binary-instructions). Then run it with the following command:\n\n```shell\nchmod +x ./llamafile\n./llamafile\n```\n> [Reference](https://continue.dev/docs/reference/Model%20Providers/llamafile)",
             "### Mistral API\n\nTo get access to the Mistral API, obtain your API key from the [Mistral platform](https://docs.mistral.ai/)",
-            "### DeepInfra\n\n> [Reference](https://docs.continue.dev/reference/Model%20Providers/deepinfra)"
+            "### DeepInfra\n\n> [Reference](https://docs.continue.dev/reference/Model%20Providers/deepinfra)",
+            "### Continue Proxy\n\nContinue Enterprise users can use the Continue Proxy as a single point of access to models."
           ],
           "type": "string"
         },

extensions/intellij/src/main/resources/config_schema.json

@@ -156,7 +156,8 @@
             "mistral",
             "deepinfra",
             "flowise",
-            "groq"
+            "groq",
+            "continue-proxy"
           ],
           "markdownEnumDescriptions": [
             "### OpenAI\nUse gpt-4, gpt-3.5-turbo, or any other OpenAI model. See [here](https://openai.com/product#made-for-developers) to obtain an API key.\n\n> [Reference](https://docs.continue.dev/reference/Model%20Providers/openai)",
@@ -174,7 +175,8 @@
             "### LMStudio\nLMStudio provides a professional and well-designed GUI for exploring, configuring, and serving LLMs. It is available on both Mac and Windows. To get started:\n1. Download from [lmstudio.ai](https://lmstudio.ai/) and open the application\n2. Search for and download the desired model from the home screen of LMStudio.\n3. In the left-bar, click the '<->' icon to open the Local Inference Server and press 'Start Server'.\n4. Once your model is loaded and the server has started, you can begin using Continue.\n> [Reference](https://continue.dev/docs/reference/Model%20Providers/lmstudio)",
             "### Llamafile\nTo get started with llamafiles, find and download a binary on their [GitHub repo](https://github.com/Mozilla-Ocho/llamafile#binary-instructions). Then run it with the following command:\n\n```shell\nchmod +x ./llamafile\n./llamafile\n```\n> [Reference](https://continue.dev/docs/reference/Model%20Providers/llamafile)",
             "### Mistral API\n\nTo get access to the Mistral API, obtain your API key from the [Mistral platform](https://docs.mistral.ai/)",
-            "### DeepInfra\n\n> [Reference](https://docs.continue.dev/reference/Model%20Providers/deepinfra)"
+            "### DeepInfra\n\n> [Reference](https://docs.continue.dev/reference/Model%20Providers/deepinfra)",
+            "### Continue Proxy\n\nContinue Enterprise users can use the Continue Proxy as a single point of access to models."
           ],
           "type": "string"
         },

extensions/vscode/config_schema.json

@@ -156,7 +156,8 @@
             "mistral",
             "deepinfra",
             "flowise",
-            "groq"
+            "groq",
+            "continue-proxy"
           ],
           "markdownEnumDescriptions": [
             "### OpenAI\nUse gpt-4, gpt-3.5-turbo, or any other OpenAI model. See [here](https://openai.com/product#made-for-developers) to obtain an API key.\n\n> [Reference](https://docs.continue.dev/reference/Model%20Providers/openai)",
@@ -174,7 +175,8 @@
             "### LMStudio\nLMStudio provides a professional and well-designed GUI for exploring, configuring, and serving LLMs. It is available on both Mac and Windows. To get started:\n1. Download from [lmstudio.ai](https://lmstudio.ai/) and open the application\n2. Search for and download the desired model from the home screen of LMStudio.\n3. In the left-bar, click the '<->' icon to open the Local Inference Server and press 'Start Server'.\n4. Once your model is loaded and the server has started, you can begin using Continue.\n> [Reference](https://continue.dev/docs/reference/Model%20Providers/lmstudio)",
             "### Llamafile\nTo get started with llamafiles, find and download a binary on their [GitHub repo](https://github.com/Mozilla-Ocho/llamafile#binary-instructions). Then run it with the following command:\n\n```shell\nchmod +x ./llamafile\n./llamafile\n```\n> [Reference](https://continue.dev/docs/reference/Model%20Providers/llamafile)",
             "### Mistral API\n\nTo get access to the Mistral API, obtain your API key from the [Mistral platform](https://docs.mistral.ai/)",
-            "### DeepInfra\n\n> [Reference](https://docs.continue.dev/reference/Model%20Providers/deepinfra)"
+            "### DeepInfra\n\n> [Reference](https://docs.continue.dev/reference/Model%20Providers/deepinfra)",
+            "### Continue Proxy\n\nContinue Enterprise users can use the Continue Proxy as a single point of access to models."
           ],
           "type": "string"
         },