Merge pull request #160 from lgrammel/lg/fix-llama-cpp

Add flag for parallelizable embedding model calls. #156

src/model-function/embed/EmbeddingModel.ts

@@ -18,6 +18,11 @@ export interface EmbeddingModel<
    */
   readonly maxValuesPerCall: number | undefined;
 
+  /**
+   * True if the model can handle multiple embedding calls in parallel.
+   */
+  readonly isParallizable: boolean;
+
   doEmbedValues(
     values: VALUE[],
     options?: FunctionOptions

src/model-function/embed/embed.ts

@@ -48,11 +48,21 @@ export function embedMany<VALUE>(
           }
         }
 
-        const responses = await Promise.all(
-          valueGroups.map((valueGroup) =>
-            model.doEmbedValues(valueGroup, options)
-          )
-        );
+        // call the model for each group:
+        let responses: Array<{ response: unknown; embeddings: Vector[] }>;
+        if (model.isParallizable) {
+          responses = await Promise.all(
+            valueGroups.map((valueGroup) =>
+              model.doEmbedValues(valueGroup, options)
+            )
+          );
+        } else {
+          responses = [];
+          for (const valueGroup of valueGroups) {
+            const response = await model.doEmbedValues(valueGroup, options);
+            responses.push(response);
+          }
+        }
 
         const rawResponses = responses.map((response) => response.response);
         const embeddings: Array<Vector> = [];

src/model-provider/cohere/CohereTextEmbeddingModel.ts

@@ -82,6 +82,7 @@ export class CohereTextEmbeddingModel
   }
 
   readonly maxValuesPerCall = 96;
+  readonly isParallizable = true;
   readonly embeddingDimensions: number;
 
   readonly contextWindowSize: number;

src/model-provider/huggingface/HuggingFaceTextEmbeddingModel.ts

@@ -67,6 +67,7 @@ export class HuggingFaceTextEmbeddingModel
   }
 
   readonly maxValuesPerCall;
+  readonly isParallizable = true;
 
   readonly contextWindowSize = undefined;
   readonly embeddingDimensions;

src/model-provider/llamacpp/LlamaCppTextEmbeddingModel.ts

@@ -19,6 +19,7 @@ export interface LlamaCppTextEmbeddingModelSettings
   extends EmbeddingModelSettings {
   api?: ApiConfiguration;
   embeddingDimensions?: number;
+  isParallizable?: boolean;
 }
 
 export class LlamaCppTextEmbeddingModel
@@ -38,6 +39,9 @@ export class LlamaCppTextEmbeddingModel
   }
 
   readonly maxValuesPerCall = 1;
+  get isParallizable() {
+    return this.settings.isParallizable ?? false;
+  }
 
   readonly contextWindowSize = undefined;
   readonly embeddingDimensions;

src/model-provider/openai/OpenAITextEmbeddingModel.ts

@@ -92,6 +92,7 @@ export class OpenAITextEmbeddingModel
   }
 
   readonly maxValuesPerCall = 2048;
+  readonly isParallizable = true;
 
   readonly embeddingDimensions: number;