From 3f4b3723995685f54074f7ec9784a2488f64db07 Mon Sep 17 00:00:00 2001 From: Jonathan Amsterdam Date: Tue, 25 Jun 2024 08:02:26 -0400 Subject: [PATCH 1/2] [Go] weave models.md See #469. --- docs-go/generate.sh | 5 +- docs-go/models | 158 +++++++++++++++++++++++++ docs-go/models.md | 149 ++++++++++++++---------- go/internal/doc-snippets/models.go | 181 +++++++++++++++++++++++++++++ 4 files changed, 427 insertions(+), 66 deletions(-) create mode 100644 docs-go/models diff --git a/docs-go/generate.sh b/docs-go/generate.sh index 44e011c5b..b9ef0bcbe 100755 --- a/docs-go/generate.sh +++ b/docs-go/generate.sh @@ -6,6 +6,7 @@ if [[ ! -f $weave ]]; then go -C ../go install ./internal/cmd/weave fi -$weave flows > flows.md - +for file in flows models; do + $weave $file > $file.md +done diff --git a/docs-go/models b/docs-go/models new file mode 100644 index 000000000..b01257178 --- /dev/null +++ b/docs-go/models @@ -0,0 +1,158 @@ +# Generating content + +Firebase Genkit provides an easy interface for generating content with LLMs. + +## Models + +Models in Firebase Genkit are libraries and abstractions that provide access to +various Google and non-Google LLMs. + +Models are fully instrumented for observability and come with tooling +integrations provided by the Genkit Developer UI -- you can try any model using +the model runner. + +When working with models in Genkit, you first need to configure the model you +want to work with. Model configuration is performed by the plugin system. In +this example you are configuring the Vertex AI plugin, which provides Gemini +models. + +- {Go} + + %include ../go/internal/doc-snippets/models.go import + + %include ../go/internal/doc-snippets/models.go init + +Note: Different plugins and models use different methods of +authentication. For example, Vertex API uses the Google Auth Library so it can +pull required credentials using Application Default Credentials. + +To use models provided by the plugin, you need a reference to the specific model +and version: + +- {Go} + + %include ../go/internal/doc-snippets/models.go model + +## Supported models + +Genkit provides model support through its plugin system. The following plugins +are officially supported: + +| Plugin | Models | +| ------------------------- | ------------------------------------------------------------------------ | +| [Google Generative AI][1] | Gemini Pro, Gemini Pro Vision | +| [Google Vertex AI][2] | Gemini Pro, Gemini Pro Vision, Gemini 1.5 Flash, Gemini 1.5 Pro, Imagen2 | +| [Ollama][3] | Many local models, including Gemma, Llama 2, Mistral, and more | + +[1]: plugins/google-genai.md +[2]: plugins/vertex-ai.md +[3]: plugins/ollama.md + +See the docs for each plugin for setup and usage information. + + + +## How to generate content + +Genkit provides a simple helper function for generating content with models. + +To just call the model: + +- {Go} + + %include ../go/internal/doc-snippets/models.go call + +You can pass options along with the model call. The options that are supported +depend on the model and its API. + +- {Go} + + %include ../go/internal/doc-snippets/models.go options + +### Streaming responses + +Genkit supports chunked streaming of model responses: + +- {Go} + + To use chunked streaming, pass a callback function to `Generate()`: + + %include ../go/internal/doc-snippets/models.go streaming + +## Multimodal input + +If the model supports multimodal input, you can pass image prompts: + +- {Go} + + %include ../go/internal/doc-snippets/models.go multimodal + + + +The exact format of the image prompt (`https` URL, `gs` URL, `data` URI) is +model-dependent. + +## Function calling (tools) + +Genkit models provide an interface for function calling, for models that support +it. + +- {Go} + + %include ../go/internal/doc-snippets/models.go tools + +This will automatically call the tools in order to fulfill the user prompt. + + + + + +### Recording message history + +Genkit models support maintaining a history of the messages sent to the model +and its responses, which you can use to build interactive experiences, such as +chatbots. + +- {Go} + + In the first prompt of a session, the "history" is simply the user prompt: + + %include ../go/internal/doc-snippets/models.go hist1 + + When you get a response, add it to the history: + + %include ../go/internal/doc-snippets/models.go hist2 + + You can serialize this history and persist it in a database or session storage. + For subsequent user prompts, add them to the history before calling + `Generate()`: + + %include ../go/internal/doc-snippets/models.go hist3 + +If the model you're using supports the system role, you can use the initial +history to set the system message: + +- {Go} + + %include ../go/internal/doc-snippets/models.go hist4 diff --git a/docs-go/models.md b/docs-go/models.md index 9af6a52b3..b5c6758c6 100644 --- a/docs-go/models.md +++ b/docs-go/models.md @@ -1,3 +1,5 @@ + + # Generating content Firebase Genkit provides an easy interface for generating content with LLMs. @@ -19,12 +21,15 @@ models. - {Go} ```go + import "github.com/firebase/genkit/go/ai" import "github.com/firebase/genkit/go/plugins/vertexai" ``` ```go projectID := os.Getenv("GCLOUD_PROJECT") - err := vertexai.Init(context.Background(), projectID, "us-central1") + if err := vertexai.Init(ctx, projectID, "us-central1"); err != nil { + return err + } ``` Note: Different plugins and models use different methods of @@ -37,7 +42,7 @@ and version: - {Go} ```go - gemini15pro := googleai.Model("gemini-1.5-pro") + gemini15pro := vertexai.Model("gemini-1.5-pro") ``` ## Supported models @@ -70,11 +75,17 @@ To just call the model: ```go request := ai.GenerateRequest{Messages: []*ai.Message{ - {Content: []*ai.Part{ai.NewTextPart("Tell me a joke.")}}, + {Content: []*ai.Part{ai.NewTextPart("Tell me a joke.")}}, }} - gemini15pro.Generate(context.Background(), &request, nil) - + response, err := gemini15pro.Generate(ctx, &request, nil) + if err != nil { + return err + } + responseText, err := response.Text() + if err != nil { + return err + } fmt.Println(responseText) ``` @@ -85,14 +96,14 @@ depend on the model and its API. ```go request := ai.GenerateRequest{ - Messages: []*ai.Message{ - {Content: []*ai.Part{ai.NewTextPart("Tell me a joke about dogs.")}}, - }, - Config: ai.GenerationCommonConfig{ - Temperature: 1.67, - StopSequences: []string{"abc"}, - MaxOutputTokens: 3, - }, + Messages: []*ai.Message{ + {Content: []*ai.Part{ai.NewTextPart("Tell me a joke about dogs.")}}, + }, + Config: ai.GenerationCommonConfig{ + Temperature: 1.67, + StopSequences: []string{"abc"}, + MaxOutputTokens: 3, + }, } ``` @@ -106,24 +117,31 @@ Genkit supports chunked streaming of model responses: ```go request := ai.GenerateRequest{Messages: []*ai.Message{ - {Content: []*ai.Part{ai.NewTextPart("Tell a long story about robots and ninjas.")}}, + {Content: []*ai.Part{ai.NewTextPart("Tell a long story about robots and ninjas.")}}, }} response, err := gemini15pro.Generate( - context.Background(), - &request, - func(ctx context.Context, grc *ai.GenerateResponseChunk) error { - text, err := grc.Text() - if err == nil { - fmt.Printf("Chunk: %s\n", text) - } - return err - }) - + ctx, + &request, + func(ctx context.Context, grc *ai.GenerateResponseChunk) error { + text, err := grc.Text() + if err != nil { + return err + } + fmt.Printf("Chunk: %s\n", text) + return nil + }) + if err != nil { + return err + } + // You can also still get the full response. responseText, err := response.Text() + if err != nil { + return err + } fmt.Println(responseText) ``` - + ## Multimodal input If the model supports multimodal input, you can pass image prompts: @@ -132,15 +150,18 @@ If the model supports multimodal input, you can pass image prompts: ```go imageBytes, err := os.ReadFile("img.jpg") + if err != nil { + return err + } encodedImage := base64.StdEncoding.EncodeToString(imageBytes) - + request := ai.GenerateRequest{Messages: []*ai.Message{ - {Content: []*ai.Part{ - ai.NewTextPart("Describe the following image."), - ai.NewMediaPart("", "data:image/jpeg;base64,"+encodedImage), - }}, + {Content: []*ai.Part{ + ai.NewTextPart("Describe the following image."), + ai.NewMediaPart("", "data:image/jpeg;base64,"+encodedImage), + }}, }} - gemini15pro.Generate(context.Background(), &request, nil) + gemini15pro.Generate(ctx, &request, nil) ``` @@ -157,29 +178,29 @@ it. ```go myJoke := &ai.ToolDefinition{ - Name: "myJoke", - Description: "useful when you need a joke to tell", - InputSchema: make(map[string]any), - OutputSchema: map[string]any{ - "joke": "string", - }, + Name: "myJoke", + Description: "useful when you need a joke to tell", + InputSchema: make(map[string]any), + OutputSchema: map[string]any{ + "joke": "string", + }, } ai.DefineTool( - myJoke, - nil, - func(ctx context.Context, input map[string]any) (map[string]any, error) { - return map[string]any{"joke": "haha Just kidding no joke! got you"}, nil - }, + myJoke, + nil, + func(ctx context.Context, input map[string]any) (map[string]any, error) { + return map[string]any{"joke": "haha Just kidding no joke! got you"}, nil + }, ) - + request := ai.GenerateRequest{ - Messages: []*ai.Message{ - {Content: []*ai.Part{ai.NewTextPart("Tell me a joke.")}, - Role: ai.RoleUser}, - }, - Tools: []*ai.ToolDefinition{myJoke}, + Messages: []*ai.Message{ + {Content: []*ai.Part{ai.NewTextPart("Tell me a joke.")}, + Role: ai.RoleUser}, + }, + Tools: []*ai.ToolDefinition{myJoke}, } - gemini15pro.Generate(context.Background(), &request, nil) + response, err := gemini15pro.Generate(ctx, &request, nil) ``` This will automatically call the tools in order to fulfill the user prompt. @@ -219,13 +240,13 @@ chatbots. In the first prompt of a session, the "history" is simply the user prompt: ```go - history := []*ai.Message{ - Content: []*ai.Part{ai.NewTextPart(prompt)}, - Role: ai.RoleUser, - } - + history := []*ai.Message{{ + Content: []*ai.Part{ai.NewTextPart(prompt)}, + Role: ai.RoleUser, + }} + request := ai.GenerateRequest{Messages: history} - gemini15pro.Generate(context.Background(), &request, nil) + response, err := gemini15pro.Generate(context.Background(), &request, nil) ``` When you get a response, add it to the history: @@ -240,12 +261,12 @@ chatbots. ```go history = append(history, &ai.Message{ - Content: []*ai.Part{ai.NewTextPart(prompt)}, - Role: ai.RoleUser, + Content: []*ai.Part{ai.NewTextPart(prompt)}, + Role: ai.RoleUser, }) - - request := ai.GenerateRequest{Messages: history} - gemini15pro.Generate(context.Background(), &request, nil) + + request = ai.GenerateRequest{Messages: history} + response, err = gemini15pro.Generate(ctx, &request, nil) ``` If the model you're using supports the system role, you can use the initial @@ -254,8 +275,8 @@ history to set the system message: - {Go} ```go - history := []&ai.Message{ - Content: []*ai.Part{ai.NewTextPart("Talk like a pirate.")}, - Role: ai.RoleSystem, - } + history = []*ai.Message{{ + Content: []*ai.Part{ai.NewTextPart("Talk like a pirate.")}, + Role: ai.RoleSystem, + }} ``` diff --git a/go/internal/doc-snippets/models.go b/go/internal/doc-snippets/models.go index 6fdd3f3b5..e7258b6f5 100644 --- a/go/internal/doc-snippets/models.go +++ b/go/internal/doc-snippets/models.go @@ -13,3 +13,184 @@ // limitations under the License. package snippets + +import ( + "context" + "encoding/base64" + "fmt" + "os" +) + +// !+import +import "github.com/firebase/genkit/go/ai" +import "github.com/firebase/genkit/go/plugins/vertexai" + +// !-import + +// Globals for simplification only. +// Bad style: don't do this. +var ctx = context.Background() +var gemini15pro *ai.Model + +func m1() error { + // !+init + projectID := os.Getenv("GCLOUD_PROJECT") + if err := vertexai.Init(ctx, projectID, "us-central1"); err != nil { + return err + } + // !-init + _ = projectID + + // !+model + gemini15pro := vertexai.Model("gemini-1.5-pro") + // !-model + + // !+call + request := ai.GenerateRequest{Messages: []*ai.Message{ + {Content: []*ai.Part{ai.NewTextPart("Tell me a joke.")}}, + }} + response, err := gemini15pro.Generate(ctx, &request, nil) + if err != nil { + return err + } + + responseText, err := response.Text() + if err != nil { + return err + } + fmt.Println(responseText) + // !-call + return nil +} + +func opts() error { + // !+options + request := ai.GenerateRequest{ + Messages: []*ai.Message{ + {Content: []*ai.Part{ai.NewTextPart("Tell me a joke about dogs.")}}, + }, + Config: ai.GenerationCommonConfig{ + Temperature: 1.67, + StopSequences: []string{"abc"}, + MaxOutputTokens: 3, + }, + } + // !-options + _ = request + return nil +} + +func streaming() error { + // !+streaming + request := ai.GenerateRequest{Messages: []*ai.Message{ + {Content: []*ai.Part{ai.NewTextPart("Tell a long story about robots and ninjas.")}}, + }} + response, err := gemini15pro.Generate( + ctx, + &request, + func(ctx context.Context, grc *ai.GenerateResponseChunk) error { + text, err := grc.Text() + if err != nil { + return err + } + fmt.Printf("Chunk: %s\n", text) + return nil + }) + if err != nil { + return err + } + + // You can also still get the full response. + responseText, err := response.Text() + if err != nil { + return err + } + fmt.Println(responseText) + + // !-streaming + return nil +} + +func multi() error { + // !+multimodal + imageBytes, err := os.ReadFile("img.jpg") + if err != nil { + return err + } + encodedImage := base64.StdEncoding.EncodeToString(imageBytes) + + request := ai.GenerateRequest{Messages: []*ai.Message{ + {Content: []*ai.Part{ + ai.NewTextPart("Describe the following image."), + ai.NewMediaPart("", "data:image/jpeg;base64,"+encodedImage), + }}, + }} + gemini15pro.Generate(ctx, &request, nil) + // !-multimodal + return nil +} + +func tools() error { + // !+tools + myJoke := &ai.ToolDefinition{ + Name: "myJoke", + Description: "useful when you need a joke to tell", + InputSchema: make(map[string]any), + OutputSchema: map[string]any{ + "joke": "string", + }, + } + ai.DefineTool( + myJoke, + nil, + func(ctx context.Context, input map[string]any) (map[string]any, error) { + return map[string]any{"joke": "haha Just kidding no joke! got you"}, nil + }, + ) + + request := ai.GenerateRequest{ + Messages: []*ai.Message{ + {Content: []*ai.Part{ai.NewTextPart("Tell me a joke.")}, + Role: ai.RoleUser}, + }, + Tools: []*ai.ToolDefinition{myJoke}, + } + response, err := gemini15pro.Generate(ctx, &request, nil) + // !-tools + _ = response + return err +} + +func history() error { + var prompt string + // !+hist1 + history := []*ai.Message{{ + Content: []*ai.Part{ai.NewTextPart(prompt)}, + Role: ai.RoleUser, + }} + + request := ai.GenerateRequest{Messages: history} + response, err := gemini15pro.Generate(context.Background(), &request, nil) + // !-hist1 + _ = err + // !+hist2 + history = append(history, response.Candidates[0].Message) + // !-hist2 + + // !+hist3 + history = append(history, &ai.Message{ + Content: []*ai.Part{ai.NewTextPart(prompt)}, + Role: ai.RoleUser, + }) + + request = ai.GenerateRequest{Messages: history} + response, err = gemini15pro.Generate(ctx, &request, nil) + // !-hist3 + // !+hist4 + history = []*ai.Message{{ + Content: []*ai.Part{ai.NewTextPart("Talk like a pirate.")}, + Role: ai.RoleSystem, + }} + // !-hist4 + return nil +} From ba3bcbf0877bbd4694eddcff69ec95b4963adf3d Mon Sep 17 00:00:00 2001 From: Jonathan Amsterdam Date: Tue, 25 Jun 2024 12:26:13 -0400 Subject: [PATCH 2/2] fix formatting --- docs-go/models | 3 ++- docs-go/models.md | 15 ++++++++------- go/internal/cmd/weave/weave.go | 6 ++++-- 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/docs-go/models b/docs-go/models index b01257178..1a8e2cc21 100644 --- a/docs-go/models +++ b/docs-go/models @@ -79,7 +79,7 @@ Genkit supports chunked streaming of model responses: To use chunked streaming, pass a callback function to `Generate()`: %include ../go/internal/doc-snippets/models.go streaming - + ## Multimodal input If the model supports multimodal input, you can pass image prompts: @@ -119,6 +119,7 @@ const docs = await companyPolicyRetriever({ query: question }); await generate({ model: geminiPro, prompt: `Answer using the available context from company policy: ${question}`, + context: docs, }); ``` diff --git a/docs-go/models.md b/docs-go/models.md index b5c6758c6..ed41d04d5 100644 --- a/docs-go/models.md +++ b/docs-go/models.md @@ -81,7 +81,7 @@ To just call the model: if err != nil { return err } - + responseText, err := response.Text() if err != nil { return err @@ -133,7 +133,7 @@ Genkit supports chunked streaming of model responses: if err != nil { return err } - + // You can also still get the full response. responseText, err := response.Text() if err != nil { @@ -141,7 +141,7 @@ Genkit supports chunked streaming of model responses: } fmt.Println(responseText) ``` - + ## Multimodal input If the model supports multimodal input, you can pass image prompts: @@ -154,7 +154,7 @@ If the model supports multimodal input, you can pass image prompts: return err } encodedImage := base64.StdEncoding.EncodeToString(imageBytes) - + request := ai.GenerateRequest{Messages: []*ai.Message{ {Content: []*ai.Part{ ai.NewTextPart("Describe the following image."), @@ -192,7 +192,7 @@ it. return map[string]any{"joke": "haha Just kidding no joke! got you"}, nil }, ) - + request := ai.GenerateRequest{ Messages: []*ai.Message{ {Content: []*ai.Part{ai.NewTextPart("Tell me a joke.")}, @@ -220,6 +220,7 @@ const docs = await companyPolicyRetriever({ query: question }); await generate({ model: geminiPro, prompt: `Answer using the available context from company policy: ${question}`, + context: docs, }); ``` @@ -244,7 +245,7 @@ chatbots. Content: []*ai.Part{ai.NewTextPart(prompt)}, Role: ai.RoleUser, }} - + request := ai.GenerateRequest{Messages: history} response, err := gemini15pro.Generate(context.Background(), &request, nil) ``` @@ -264,7 +265,7 @@ chatbots. Content: []*ai.Part{ai.NewTextPart(prompt)}, Role: ai.RoleUser, }) - + request = ai.GenerateRequest{Messages: history} response, err = gemini15pro.Generate(ctx, &request, nil) ``` diff --git a/go/internal/cmd/weave/weave.go b/go/internal/cmd/weave/weave.go index 14e6450d8..71a1cc72b 100644 --- a/go/internal/cmd/weave/weave.go +++ b/go/internal/cmd/weave/weave.go @@ -180,7 +180,7 @@ func indented(line string) bool { // cleanListing removes entirely blank leading and trailing lines from // text, and removes n leading tabs. -// It then prefixes each line with indent. +// It then prefixes each non-blank line with indent. func cleanListing(text, indent string) string { lines := strings.Split(text, "\n") @@ -212,7 +212,9 @@ func cleanListing(text, indent string) string { } // add indent for i, ln := range lines { - lines[i] = indent + ln + if ln != "" { + lines[i] = indent + ln + } } return strings.Join(lines, "\n") }