Conversation API: add cache support, add huggingface+mistral models (#…

…3567)

Signed-off-by: yaron2 <[email protected]>

conversation/anthropic/anthropic.go

@@ -28,16 +28,11 @@ import (
 )
 
 type Anthropic struct {
-	llm *anthropic.LLM
+	llm llms.Model
 
 	logger logger.Logger
 }
 
-type AnthropicMetadata struct {
-	Key   string `json:"key"`
-	Model string `json:"model"`
-}
-
 func NewAnthropic(logger logger.Logger) conversation.Conversation {
 	a := &Anthropic{
 		logger: logger,
@@ -49,7 +44,7 @@ func NewAnthropic(logger logger.Logger) conversation.Conversation {
 const defaultModel = "claude-3-5-sonnet-20240620"
 
 func (a *Anthropic) Init(ctx context.Context, meta conversation.Metadata) error {
-	m := AnthropicMetadata{}
+	m := conversation.LangchainMetadata{}
 	err := kmeta.DecodeMetadata(meta.Properties, &m)
 	if err != nil {
 		return err
@@ -69,11 +64,21 @@ func (a *Anthropic) Init(ctx context.Context, meta conversation.Metadata) error
 	}
 
 	a.llm = llm
+
+	if m.CacheTTL != "" {
+		cachedModel, cacheErr := conversation.CacheModel(ctx, m.CacheTTL, a.llm)
+		if cacheErr != nil {
+			return cacheErr
+		}
+
+		a.llm = cachedModel
+	}
+
 	return nil
 }
 
 func (a *Anthropic) GetComponentMetadata() (metadataInfo metadata.MetadataMap) {
-	metadataStruct := AnthropicMetadata{}
+	metadataStruct := conversation.LangchainMetadata{}
 	metadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, metadata.ConversationType)
 	return
 }

conversation/anthropic/metadata.yaml

@@ -27,3 +27,9 @@ metadata:
       The Anthropic LLM to use. Defaults to claude-3-5-sonnet-20240620
     type: string
     example: 'claude-3-5-sonnet-20240620'
+  - name: cacheTTL
+    required: false
+    description: |
+      A time-to-live value for a prompt cache to expire. Uses Golang durations
+    type: string
+    example: '10m'

conversation/aws/bedrock/bedrock.go

@@ -31,7 +31,7 @@ import (
 
 type AWSBedrock struct {
 	model string
-	llm   *bedrock.LLM
+	llm   llms.Model
 
 	logger logger.Logger
 }
@@ -43,6 +43,7 @@ type AWSBedrockMetadata struct {
 	SecretKey    string `json:"secretKey"`
 	SessionToken string `json:"sessionToken"`
 	Model        string `json:"model"`
+	CacheTTL     string `json:"cacheTTL"`
 }
 
 func NewAWSBedrock(logger logger.Logger) conversation.Conversation {
@@ -81,6 +82,15 @@ func (b *AWSBedrock) Init(ctx context.Context, meta conversation.Metadata) error
 	}
 
 	b.llm = llm
+
+	if m.CacheTTL != "" {
+		cachedModel, cacheErr := conversation.CacheModel(ctx, m.CacheTTL, b.llm)
+		if cacheErr != nil {
+			return cacheErr
+		}
+
+		b.llm = cachedModel
+	}
 	return nil
 }
 

conversation/aws/bedrock/metadata.yaml

@@ -24,3 +24,9 @@ metadata:
       The LLM to use. Defaults to Bedrock's default provider model from Amazon.
     type: string
     example: 'amazon.titan-text-express-v1'
+  - name: cacheTTL
+    required: false
+    description: |
+      A time-to-live value for a prompt cache to expire. Uses Golang durations
+    type: string
+    example: '10m'

conversation/huggingface/huggingface.go

@@ -0,0 +1,129 @@
+/*
+Copyright 2024 The Dapr Authors
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package huggingface
+
+import (
+	"context"
+	"reflect"
+
+	"github.com/dapr/components-contrib/conversation"
+	"github.com/dapr/components-contrib/metadata"
+	"github.com/dapr/kit/logger"
+	kmeta "github.com/dapr/kit/metadata"
+
+	"github.com/tmc/langchaingo/llms"
+	"github.com/tmc/langchaingo/llms/huggingface"
+)
+
+type Huggingface struct {
+	llm llms.Model
+
+	logger logger.Logger
+}
+
+func NewHuggingface(logger logger.Logger) conversation.Conversation {
+	h := &Huggingface{
+		logger: logger,
+	}
+
+	return h
+}
+
+const defaultModel = "meta-llama/Meta-Llama-3-8B"
+
+func (h *Huggingface) Init(ctx context.Context, meta conversation.Metadata) error {
+	m := conversation.LangchainMetadata{}
+	err := kmeta.DecodeMetadata(meta.Properties, &m)
+	if err != nil {
+		return err
+	}
+
+	model := defaultModel
+	if m.Model != "" {
+		model = m.Model
+	}
+
+	llm, err := huggingface.New(
+		huggingface.WithModel(model),
+		huggingface.WithToken(m.Key),
+	)
+	if err != nil {
+		return err
+	}
+
+	h.llm = llm
+
+	if m.CacheTTL != "" {
+		cachedModel, cacheErr := conversation.CacheModel(ctx, m.CacheTTL, h.llm)
+		if cacheErr != nil {
+			return cacheErr
+		}
+
+		h.llm = cachedModel
+	}
+
+	return nil
+}
+
+func (h *Huggingface) GetComponentMetadata() (metadataInfo metadata.MetadataMap) {
+	metadataStruct := conversation.LangchainMetadata{}
+	metadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, metadata.ConversationType)
+	return
+}
+
+func (h *Huggingface) Converse(ctx context.Context, r *conversation.ConversationRequest) (res *conversation.ConversationResponse, err error) {
+	messages := make([]llms.MessageContent, 0, len(r.Inputs))
+
+	for _, input := range r.Inputs {
+		role := conversation.ConvertLangchainRole(input.Role)
+
+		messages = append(messages, llms.MessageContent{
+			Role: role,
+			Parts: []llms.ContentPart{
+				llms.TextPart(input.Message),
+			},
+		})
+	}
+
+	opts := []llms.CallOption{}
+
+	if r.Temperature > 0 {
+		opts = append(opts, conversation.LangchainTemperature(r.Temperature))
+	}
+
+	resp, err := h.llm.GenerateContent(ctx, messages, opts...)
+	if err != nil {
+		return nil, err
+	}
+
+	outputs := make([]conversation.ConversationResult, 0, len(resp.Choices))
+
+	for i := range resp.Choices {
+		outputs = append(outputs, conversation.ConversationResult{
+			Result:     resp.Choices[i].Content,
+			Parameters: r.Parameters,
+		})
+	}
+
+	res = &conversation.ConversationResponse{
+		Outputs: outputs,
+	}
+
+	return res, nil
+}
+
+func (h *Huggingface) Close() error {
+	return nil
+}

conversation/huggingface/metadata.yaml

@@ -0,0 +1,35 @@
+# yaml-language-server: $schema=../../../component-metadata-schema.json
+schemaVersion: v1
+type: conversation
+name: huggingface
+version: v1
+status: alpha
+title: "Huggingface"
+urls:
+  - title: Reference
+    url: https://docs.dapr.io/reference/components-reference/supported-conversation/setup-huggingface/
+authenticationProfiles:
+  - title: "API Key"
+    description: "Authenticate using an API key"
+    metadata:
+      - name: key
+        type: string
+        required: true
+        sensitive: true
+        description: |
+          API key for Huggingface.
+        example:  "**********"
+        default: ""
+metadata:
+  - name: model
+    required: false
+    description: |
+      The Huggingface LLM to use. Defaults to meta-llama/Meta-Llama-3-8B
+    type: string
+    example: 'meta-llama/Meta-Llama-3-8B'
+  - name: cacheTTL
+    required: false
+    description: |
+      A time-to-live value for a prompt cache to expire. Uses Golang durations
+    type: string
+    example: '10m'

conversation/metadata.go

@@ -20,3 +20,10 @@ import "github.com/dapr/components-contrib/metadata"
 type Metadata struct {
 	metadata.Base `json:",inline"`
 }
+
+// LangchainMetadata is a common metadata structure for langchain supported implementations.
+type LangchainMetadata struct {
+	Key      string `json:"key"`
+	Model    string `json:"model"`
+	CacheTTL string `json:"cacheTTL"`
+}

conversation/mistral/metadata.yaml

@@ -0,0 +1,35 @@
+# yaml-language-server: $schema=../../../component-metadata-schema.json
+schemaVersion: v1
+type: conversation
+name: mistral
+version: v1
+status: alpha
+title: "Mistral"
+urls:
+  - title: Reference
+    url: https://docs.dapr.io/reference/components-reference/supported-conversation/setup-mistral/
+authenticationProfiles:
+  - title: "API Key"
+    description: "Authenticate using an API key"
+    metadata:
+      - name: key
+        type: string
+        required: true
+        sensitive: true
+        description: |
+          API key for Mistral.
+        example:  "**********"
+        default: ""
+metadata:
+  - name: model
+    required: false
+    description: |
+      The Mistral LLM to use. Defaults to open-mistral-7b
+    type: string
+    example: 'open-mistral-7b'
+  - name: cacheTTL
+    required: false
+    description: |
+      A time-to-live value for a prompt cache to expire. Uses Golang durations
+    type: string
+    example: '10m'