From c444b0fdd0f9d8a26b093f5cd81d43209267ee91 Mon Sep 17 00:00:00 2001 From: Max Date: Mon, 11 Dec 2023 20:53:47 -0700 Subject: [PATCH 1/6] #15: Update dependencies in go.mod and add input validation using validator package --- go.mod | 11 ++++++++++- go.sum | 24 ++++++++++++++++++++++-- pkg/buildAPIRequest.go | 15 ++++++++++++--- 3 files changed, 44 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index b97eba37..5b909391 100644 --- a/go.mod +++ b/go.mod @@ -1,8 +1,10 @@ module glide go 1.21.5 + require ( github.com/cloudwego/hertz v0.7.3 + github.com/go-playground/validator/v10 v10.16.0 github.com/spf13/cobra v1.8.0 go.uber.org/multierr v1.11.0 ) @@ -14,11 +16,15 @@ require ( github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect github.com/cloudwego/netpoll v0.5.0 // indirect github.com/fsnotify/fsnotify v1.5.4 // indirect + github.com/gabriel-vasile/mimetype v1.4.2 // indirect + github.com/go-playground/locales v0.14.1 // indirect + github.com/go-playground/universal-translator v0.18.1 // indirect github.com/golang/protobuf v1.5.0 // indirect github.com/henrylee2cn/ameda v1.4.10 // indirect github.com/henrylee2cn/goutil v0.0.0-20210127050712-89660552f6f8 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/klauspost/cpuid/v2 v2.0.9 // indirect + github.com/leodido/go-urn v1.2.4 // indirect github.com/nyaruka/phonenumbers v1.0.55 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/tidwall/gjson v1.14.4 // indirect @@ -26,6 +32,9 @@ require ( github.com/tidwall/pretty v1.2.0 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect golang.org/x/arch v0.0.0-20210923205945-b76863e36670 // indirect - golang.org/x/sys v0.0.0-20220412211240-33da011f77ad // indirect + golang.org/x/crypto v0.7.0 // indirect + golang.org/x/net v0.8.0 // indirect + golang.org/x/sys v0.6.0 // indirect + golang.org/x/text v0.8.0 // indirect google.golang.org/protobuf v1.27.1 // indirect ) diff --git a/go.sum b/go.sum index a10f47d5..84649808 100644 --- a/go.sum +++ b/go.sum @@ -20,6 +20,16 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI= github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= +github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU= +github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA= +github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= +github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= +github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= +github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= +github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= +github.com/go-playground/validator/v10 v10.16.0 h1:x+plE831WK4vaKHO/jpgUGsvLKIqRRkz6M78GuJAfGE= +github.com/go-playground/validator/v10 v10.16.0/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.5.0 h1:LUVKkCeviFUMKqHa4tXIIij/lbhnMbP7Fn5wKdKkRh4= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= @@ -38,6 +48,8 @@ github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q= +github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4= github.com/nyaruka/phonenumbers v1.0.55 h1:bj0nTO88Y68KeUQ/n3Lo2KgK7lM1hF7L9NFuwcCl3yg= github.com/nyaruka/phonenumbers v1.0.55/go.mod h1:sDaTZ/KPX5f8qyV9qN+hIm+4ZBARJrupC6LuhshJq1U= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -59,8 +71,9 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/tidwall/gjson v1.9.3/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/gjson v1.14.4 h1:uo0p8EbA09J7RQaflQ1aBRffTR7xedD2bcIVSYxLnkM= github.com/tidwall/gjson v1.14.4/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= @@ -76,13 +89,20 @@ golang.org/x/arch v0.0.0-20201008161808-52c3e6f60cff/go.mod h1:flIaEI6LNU6xOCD5P golang.org/x/arch v0.0.0-20210923205945-b76863e36670 h1:18EFjUmQOcUvxNYSkA6jO9VAiXCnxFY6NyDX0bHDmkU= golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.7.0 h1:AvwMYaRytfdeVt3u6mLaxYtErKYjxA2OXjJ1HHq6t3A= +golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= +golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20220110181412-a018aaa089fe/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220412211240-33da011f77ad h1:ntjMns5wyP/fN65tdBD4g8J5w8n015+iIIs9rtjXkY0= golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= +golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/pkg/buildAPIRequest.go b/pkg/buildAPIRequest.go index 6b8af6ea..07f8a96b 100644 --- a/pkg/buildAPIRequest.go +++ b/pkg/buildAPIRequest.go @@ -2,8 +2,12 @@ package pkg import ( "errors" + "github.com/go-playground/validator/v10" ) +// Create a new validator instance +var validate *validator.Validate = validator.New() + type ProviderConfigs map[string]interface{} // TODO: import from types.go func BuildAPIRequest(provider string, params map[string]string, mode string, configList map[string]interface{}) (interface{}, error) { @@ -24,9 +28,7 @@ func BuildAPIRequest(provider string, params map[string]string, mode string, con } - // TODO: Next need to build the request based on the params from the client - // First for each param in param check if present. If yes then add it to the request. - // If not & the param is required, return a default value from the provider config + // Build the providerConfig map by iterating over the keys in the providerConfig map and checking if the key exists in the params map for key := range providerConfig { if value, exists := params[key]; exists { @@ -34,6 +36,13 @@ func BuildAPIRequest(provider string, params map[string]string, mode string, con } } + // Validate the providerConfig map using the validator package + err := validate.Struct(providerConfig) + if err != nil { + // Handle validation error + return nil, err + } + } From 662f4bf32990ca026805e8b4e4baaf86be63358c Mon Sep 17 00:00:00 2001 From: Max Date: Mon, 11 Dec 2023 21:07:46 -0700 Subject: [PATCH 2/6] #15: Fix validation error in BuildAPIRequest function --- pkg/buildAPIRequest.go | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/pkg/buildAPIRequest.go b/pkg/buildAPIRequest.go index 07f8a96b..611bde51 100644 --- a/pkg/buildAPIRequest.go +++ b/pkg/buildAPIRequest.go @@ -3,6 +3,7 @@ package pkg import ( "errors" "github.com/go-playground/validator/v10" + "fmt" ) // Create a new validator instance @@ -40,14 +41,8 @@ func BuildAPIRequest(provider string, params map[string]string, mode string, con err := validate.Struct(providerConfig) if err != nil { // Handle validation error - return nil, err + return nil, fmt.Errorf("validation error: %v", err) } - - -} - - // For now, return providerConfig and nil error to satisfy the function signature + // If everything is fine, return the providerConfig and nil error return providerConfig, nil - - -} \ No newline at end of file +} From fb953867f72720dc17a9e6db1c127bd23f84cbec Mon Sep 17 00:00:00 2001 From: Max Date: Mon, 11 Dec 2023 21:29:12 -0700 Subject: [PATCH 3/6] #15: Refactor OpenAI provider configuration structure --- pkg/buildAPIRequest.go | 4 ++- pkg/providers/openai/chat.go | 47 ++++++++++++++++++----------------- pkg/providers/openai/index.go | 6 ++++- 3 files changed, 32 insertions(+), 25 deletions(-) diff --git a/pkg/buildAPIRequest.go b/pkg/buildAPIRequest.go index 611bde51..fb2dede6 100644 --- a/pkg/buildAPIRequest.go +++ b/pkg/buildAPIRequest.go @@ -4,12 +4,14 @@ import ( "errors" "github.com/go-playground/validator/v10" "fmt" + "glide/pkg/providers" ) +type ProviderConfigs = pkg.ProviderConfigs + // Create a new validator instance var validate *validator.Validate = validator.New() -type ProviderConfigs map[string]interface{} // TODO: import from types.go func BuildAPIRequest(provider string, params map[string]string, mode string, configList map[string]interface{}) (interface{}, error) { // provider is the name of the provider, e.g. "openai", params is the map of parameters from the client, diff --git a/pkg/providers/openai/chat.go b/pkg/providers/openai/chat.go index bb9051d7..9cf5d83c 100644 --- a/pkg/providers/openai/chat.go +++ b/pkg/providers/openai/chat.go @@ -1,4 +1,4 @@ -package openai +package pkg type ProviderConfig struct { Model ConfigItem `json:"model" validate:"required,lowercase"` @@ -21,6 +21,7 @@ type ProviderConfig struct { type ConfigItem struct { Param string `json:"param" validate:"required"` + Required bool `json:"required" validate:"omitempty,boolean"` Default interface{} `json:"default"` } @@ -34,69 +35,69 @@ func OpenAiChatDefaultConfig() ProviderConfig { }, Messages: ConfigItem{ Param: "messages", + Required: true, Default: "", }, - Functions: ConfigItem{ - Param: "functions", - }, - FunctionCall: ConfigItem{ - Param: "function_call", - }, - MaxTokens: NumericConfigItem{ + MaxTokens: ConfigItem{ Param: "max_tokens", + Required: false, Default: 100, - Min: 0, }, - Temperature: NumericConfigItem{ + Temperature: ConfigItem{ Param: "temperature", + Required: false, Default: 1, - Min: 0, - Max: 2, }, - TopP: NumericConfigItem{ + TopP: ConfigItem{ Param: "top_p", + Required: false, Default: 1, - Min: 0, - Max: 1, }, - N: NumericConfigItem{ + N: ConfigItem{ Param: "n", + Required: false, Default: 1, }, - Stream: BoolConfigItem{ + Stream: ConfigItem{ Param: "stream", + Required: false, Default: false, }, Stop: ConfigItem{ Param: "stop", + Required: false, }, - PresencePenalty: NumericConfigItem{ + PresencePenalty: ConfigItem{ Param: "presence_penalty", - Min: -2, - Max: 2, + Required: false, }, - FrequencyPenalty: NumericConfigItem{ + FrequencyPenalty: ConfigItem{ Param: "frequency_penalty", - Min: -2, - Max: 2, + Required: false, }, LogitBias: ConfigItem{ Param: "logit_bias", + Required: false, }, User: ConfigItem{ Param: "user", + Required: false, }, Seed: ConfigItem{ Param: "seed", + Required: false, }, Tools: ConfigItem{ Param: "tools", + Required: false, }, ToolChoice: ConfigItem{ Param: "tool_choice", + Required: false, }, ResponseFormat: ConfigItem{ Param: "response_format", + Required: false, }, } } diff --git a/pkg/providers/openai/index.go b/pkg/providers/openai/index.go index 9527b777..2fb88c1a 100644 --- a/pkg/providers/openai/index.go +++ b/pkg/providers/openai/index.go @@ -1,6 +1,10 @@ package pkg -type ProviderConfigs map[string]interface{} // TODO: import from types.go +import ( + "glide/pkg/providers" +) + +type ProviderConfigs = pkg.ProviderConfigs var OpenAIConfig = ProviderConfigs{ "api": OpenAIAPIConfig, From 86c198b46f3292183434b22374c87e5efe67c11c Mon Sep 17 00:00:00 2001 From: Max Date: Mon, 11 Dec 2023 21:38:34 -0700 Subject: [PATCH 4/6] #15: TODO: Import provider configs to the config list --- pkg/buildAPIRequest.go | 2 ++ pkg/providers/openai/index.go | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/pkg/buildAPIRequest.go b/pkg/buildAPIRequest.go index fb2dede6..a3e4bcb0 100644 --- a/pkg/buildAPIRequest.go +++ b/pkg/buildAPIRequest.go @@ -9,6 +9,8 @@ import ( type ProviderConfigs = pkg.ProviderConfigs +// TODO: import provider configs to the config list + // Create a new validator instance var validate *validator.Validate = validator.New() diff --git a/pkg/providers/openai/index.go b/pkg/providers/openai/index.go index 2fb88c1a..b5535dbd 100644 --- a/pkg/providers/openai/index.go +++ b/pkg/providers/openai/index.go @@ -4,9 +4,9 @@ import ( "glide/pkg/providers" ) -type ProviderConfigs = pkg.ProviderConfigs -var OpenAIConfig = ProviderConfigs{ +// TODO: this needs to be imported into buildAPIRequest.go +var OpenAIConfig = pkg.ProviderConfigs{ "api": OpenAIAPIConfig, "chat": OpenAiChatDefaultConfig, } From 09c3ceb60cc8ba7e437ef8249267f8245e9c392c Mon Sep 17 00:00:00 2001 From: Max Date: Mon, 11 Dec 2023 21:45:12 -0700 Subject: [PATCH 5/6] #15: Import OpenAI provider configuration and update BuildAPIRequest function --- pkg/buildAPIRequest.go | 9 +++++++-- pkg/providers/openai/api.go | 2 +- pkg/providers/openai/chat.go | 2 +- pkg/providers/openai/index.go | 2 +- 4 files changed, 10 insertions(+), 5 deletions(-) diff --git a/pkg/buildAPIRequest.go b/pkg/buildAPIRequest.go index a3e4bcb0..dd1bdfc3 100644 --- a/pkg/buildAPIRequest.go +++ b/pkg/buildAPIRequest.go @@ -5,17 +5,22 @@ import ( "github.com/go-playground/validator/v10" "fmt" "glide/pkg/providers" + "glide/pkg/providers/openai" ) type ProviderConfigs = pkg.ProviderConfigs -// TODO: import provider configs to the config list +// Initialize configList + +var configList = map[string]interface{}{ + "openai": openai.OpenAIConfig, +} // Create a new validator instance var validate *validator.Validate = validator.New() -func BuildAPIRequest(provider string, params map[string]string, mode string, configList map[string]interface{}) (interface{}, error) { +func BuildAPIRequest(provider string, params map[string]string, mode string) (interface{}, error) { // provider is the name of the provider, e.g. "openai", params is the map of parameters from the client, // mode is the mode of the provider, e.g. "chat", configList is the list of provider configurations diff --git a/pkg/providers/openai/api.go b/pkg/providers/openai/api.go index 515e2cc1..4a7a013b 100644 --- a/pkg/providers/openai/api.go +++ b/pkg/providers/openai/api.go @@ -1,4 +1,4 @@ -package pkg +package openai import ( "fmt" diff --git a/pkg/providers/openai/chat.go b/pkg/providers/openai/chat.go index 9cf5d83c..47824ef2 100644 --- a/pkg/providers/openai/chat.go +++ b/pkg/providers/openai/chat.go @@ -1,4 +1,4 @@ -package pkg +package openai type ProviderConfig struct { Model ConfigItem `json:"model" validate:"required,lowercase"` diff --git a/pkg/providers/openai/index.go b/pkg/providers/openai/index.go index b5535dbd..d7a7a60c 100644 --- a/pkg/providers/openai/index.go +++ b/pkg/providers/openai/index.go @@ -1,4 +1,4 @@ -package pkg +package openai import ( "glide/pkg/providers" From b709c84d87467bc0ceb67dc358a67de5c91e43fa Mon Sep 17 00:00:00 2001 From: Max Date: Mon, 11 Dec 2023 21:46:31 -0700 Subject: [PATCH 6/6] #15: comments --- pkg/buildAPIRequest.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/buildAPIRequest.go b/pkg/buildAPIRequest.go index dd1bdfc3..f28d22b1 100644 --- a/pkg/buildAPIRequest.go +++ b/pkg/buildAPIRequest.go @@ -1,3 +1,5 @@ +// this file contains the BuildAPIRequest function which takes in the provider name, params map, and mode and returns the providerConfig map and error +// The providerConfig map can be used to build the API request to the provider package pkg import (