diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..d46e10b9 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,5 @@ +# This file is documented at https://git-scm.com/docs/gitattributes. +# Linguist-specific attributes are documented at +# https://github.com/github/linguist. + +go.sum linguist-generated=true diff --git a/CHANGELOG.md b/CHANGELOG.md index f6c2c138..130e2f2a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,6 +41,26 @@ TBU TBU +## [0.0.3-rc2] (Apr 17, 2024) + +Final major improvements to streaming chat workflow. Fixed issues with Cohere streaming chat. +Expanded and revisited Cohere params in config. + +### Added + +- πŸ”§ #195 #196: Set router ctx in stream chunks & handle end of stream in case of some errors (@roma-glushko) +- πŸ›πŸ”§ #197: Handle max_tokens & content_filtered finish reasons across OpenAI, Azure and Cohere (@roma-glushko) + +### Changed + +- πŸ”§ πŸ’₯ #198: Expose more Cohere params & fixing validation of provider params in config (breaking change) (@roma-glushko) +- πŸ”§ #186: Rendering Durations in a human-friendly way (@roma-glushko) + +### Fixed + +- πŸ› #209: Embed Swagger specs into binary to fix panics caused by missing swagger.yaml file (@roma-glushko) +- πŸ› #200: Implemented a custom json per line stream reader to read Cohere chat streams correctly (@roma-glushko) + ## [0.0.3-rc.1] (Apr 7th, 2024) Bringing support for streaming chat in Glide. diff --git a/README.md b/README.md index 797f7409..9edbfb14 100644 --- a/README.md +++ b/README.md @@ -46,9 +46,9 @@ Check out our [documentation](https://glide.einstack.ai)! |-----------------------------------------------------------------------|-------------------------------------------| | OpenAI | βœ… Chat
βœ… Streaming Chat | | Anthropic | βœ… Chat
πŸ—οΈ Streaming Chat (coming soon) | -| Azure OpenAI | βœ… Chat
πŸ—οΈ Streaming Chat (coming soon) | +| Azure OpenAI | βœ… Chat
βœ… Streaming Chat | | AWS Bedrock (Titan) | βœ… Chat | -| Cohere | βœ… Chat
πŸ—οΈ Streaming Chat (coming soon) | +| Cohere | βœ… Chat
βœ… Streaming Chat | | Google Gemini | πŸ—οΈ Chat (coming soon) | | OctoML | βœ… Chat | | Ollama | βœ… Chat | @@ -213,8 +213,6 @@ To let you work with Glide's API with ease, we are going to provide you with SDK - Python (coming soon) - NodeJS (coming soon) -- Golang (coming soon) -- Rust (coming soon) ## Core Concepts diff --git a/docs/docs.go b/docs/docs.go index d575a50a..65d48212 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -378,20 +378,6 @@ const docTemplate = `{ } } }, - "cohere.ChatHistory": { - "type": "object", - "properties": { - "message": { - "type": "string" - }, - "role": { - "type": "string" - }, - "user": { - "type": "string" - } - } - }, "cohere.Config": { "type": "object", "required": [ @@ -410,42 +396,64 @@ const docTemplate = `{ "$ref": "#/definitions/cohere.Params" }, "model": { + "description": "https://docs.cohere.com/docs/models#command", "type": "string" } } }, "cohere.Params": { "type": "object", + "required": [ + "temperature" + ], "properties": { - "chat_history": { - "type": "array", - "items": { - "$ref": "#/definitions/cohere.ChatHistory" - } - }, - "citiation_quality": { - "type": "string" - }, "connectors": { "type": "array", "items": { "type": "string" } }, - "conversation_id": { - "type": "string" + "frequency_penalty": { + "type": "number", + "maximum": 1, + "minimum": 0 + }, + "k": { + "type": "integer", + "maximum": 500, + "minimum": 0 + }, + "max_tokens": { + "type": "integer" + }, + "p": { + "type": "number", + "maximum": 0.99, + "minimum": 0.01 }, - "preamble_override": { + "preamble": { "type": "string" }, + "presence_penalty": { + "type": "number", + "maximum": 1, + "minimum": 0 + }, "prompt_truncation": { "type": "string" }, "search_queries_only": { "type": "boolean" }, - "stream": { - "type": "boolean" + "seed": { + "type": "integer" + }, + "stop_sequences": { + "type": "array", + "maxItems": 5, + "items": { + "type": "string" + } }, "temperature": { "type": "number" diff --git a/docs/swagger.go b/docs/swagger.go new file mode 100644 index 00000000..0047c040 --- /dev/null +++ b/docs/swagger.go @@ -0,0 +1,6 @@ +package docs + +import _ "embed" + +//go:embed swagger.json +var SwaggerJSON []byte diff --git a/docs/swagger.json b/docs/swagger.json index ce439bc8..1588b4a5 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -375,20 +375,6 @@ } } }, - "cohere.ChatHistory": { - "type": "object", - "properties": { - "message": { - "type": "string" - }, - "role": { - "type": "string" - }, - "user": { - "type": "string" - } - } - }, "cohere.Config": { "type": "object", "required": [ @@ -407,42 +393,64 @@ "$ref": "#/definitions/cohere.Params" }, "model": { + "description": "https://docs.cohere.com/docs/models#command", "type": "string" } } }, "cohere.Params": { "type": "object", + "required": [ + "temperature" + ], "properties": { - "chat_history": { - "type": "array", - "items": { - "$ref": "#/definitions/cohere.ChatHistory" - } - }, - "citiation_quality": { - "type": "string" - }, "connectors": { "type": "array", "items": { "type": "string" } }, - "conversation_id": { - "type": "string" + "frequency_penalty": { + "type": "number", + "maximum": 1, + "minimum": 0 + }, + "k": { + "type": "integer", + "maximum": 500, + "minimum": 0 + }, + "max_tokens": { + "type": "integer" + }, + "p": { + "type": "number", + "maximum": 0.99, + "minimum": 0.01 }, - "preamble_override": { + "preamble": { "type": "string" }, + "presence_penalty": { + "type": "number", + "maximum": 1, + "minimum": 0 + }, "prompt_truncation": { "type": "string" }, "search_queries_only": { "type": "boolean" }, - "stream": { - "type": "boolean" + "seed": { + "type": "integer" + }, + "stop_sequences": { + "type": "array", + "maxItems": 5, + "items": { + "type": "string" + } }, "temperature": { "type": "number" diff --git a/docs/swagger.yaml b/docs/swagger.yaml index 00e800eb..9078f891 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -129,15 +129,6 @@ definitions: timeout: type: string type: object - cohere.ChatHistory: - properties: - message: - type: string - role: - type: string - user: - type: string - type: object cohere.Config: properties: baseUrl: @@ -147,6 +138,7 @@ definitions: defaultParams: $ref: '#/definitions/cohere.Params' model: + description: https://docs.cohere.com/docs/models#command type: string required: - baseUrl @@ -155,28 +147,45 @@ definitions: type: object cohere.Params: properties: - chat_history: - items: - $ref: '#/definitions/cohere.ChatHistory' - type: array - citiation_quality: - type: string connectors: items: type: string type: array - conversation_id: - type: string - preamble_override: + frequency_penalty: + maximum: 1 + minimum: 0 + type: number + k: + maximum: 500 + minimum: 0 + type: integer + max_tokens: + type: integer + p: + maximum: 0.99 + minimum: 0.01 + type: number + preamble: type: string + presence_penalty: + maximum: 1 + minimum: 0 + type: number prompt_truncation: type: string search_queries_only: type: boolean - stream: - type: boolean + seed: + type: integer + stop_sequences: + items: + type: string + maxItems: 5 + type: array temperature: type: number + required: + - temperature type: object http.ErrorSchema: properties: diff --git a/go.mod b/go.mod index c90e943c..13ccb639 100644 --- a/go.mod +++ b/go.mod @@ -9,15 +9,15 @@ require ( github.com/aws/aws-sdk-go-v2/service/bedrockruntime v1.5.6 github.com/go-playground/validator/v10 v10.17.0 github.com/gofiber/contrib/fiberzap/v2 v2.1.2 - github.com/gofiber/contrib/swagger v1.1.1 github.com/gofiber/contrib/websocket v1.3.0 github.com/gofiber/fiber/v2 v2.52.2 + github.com/gofiber/swagger v1.0.0 github.com/google/uuid v1.6.0 github.com/joho/godotenv v1.5.1 github.com/r3labs/sse/v2 v2.10.0 github.com/spf13/cobra v1.8.0 github.com/stretchr/testify v1.8.4 - github.com/swaggo/swag v1.16.2 + github.com/swaggo/swag v1.16.3 go.uber.org/goleak v1.3.0 go.uber.org/multierr v1.11.0 go.uber.org/zap v1.26.0 @@ -27,7 +27,6 @@ require ( require ( github.com/KyleBanks/depth v1.2.1 // indirect github.com/andybalholm/brotli v1.1.0 // indirect - github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.4 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.11 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.10 // indirect @@ -42,16 +41,10 @@ require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/fasthttp/websocket v1.5.7 // indirect github.com/gabriel-vasile/mimetype v1.4.2 // indirect - github.com/go-openapi/analysis v0.21.4 // indirect - github.com/go-openapi/errors v0.20.3 // indirect github.com/go-openapi/jsonpointer v0.20.2 // indirect github.com/go-openapi/jsonreference v0.20.4 // indirect - github.com/go-openapi/loads v0.21.2 // indirect - github.com/go-openapi/runtime v0.26.0 // indirect github.com/go-openapi/spec v0.20.13 // indirect - github.com/go-openapi/strfmt v0.21.7 // indirect github.com/go-openapi/swag v0.22.7 // indirect - github.com/go-openapi/validate v0.22.1 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect @@ -62,22 +55,18 @@ require ( github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-runewidth v0.0.15 // indirect - github.com/mitchellh/mapstructure v1.5.0 // indirect - github.com/oklog/ulid v1.3.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/savsgio/gotils v0.0.0-20230208104028-c358bd845dee // indirect github.com/spf13/pflag v1.0.5 // indirect - github.com/tidwall/pretty v1.2.0 // indirect + github.com/swaggo/files/v2 v2.0.0 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/valyala/fasthttp v1.52.0 // indirect github.com/valyala/tcplisten v1.0.0 // indirect - go.mongodb.org/mongo-driver v1.11.3 // indirect golang.org/x/crypto v0.21.0 // indirect golang.org/x/net v0.21.0 // indirect golang.org/x/sys v0.18.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/tools v0.16.1 // indirect gopkg.in/cenkalti/backoff.v1 v1.1.0 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect ) diff --git a/go.sum b/go.sum index c59f3570..184c944d 100644 --- a/go.sum +++ b/go.sum @@ -1,13 +1,7 @@ -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc= github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE= -github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= -github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M= github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= -github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= -github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= -github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/aws/aws-sdk-go-v2 v1.24.1 h1:xAojnj+ktS95YZlDf0zxWBkbFtymPeDP+rvUQIH3uAU= github.com/aws/aws-sdk-go-v2 v1.24.1/go.mod h1:LNh45Br1YAkEKaAqvmE1m8FUx6a5b/V0oAKV7of29b4= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.4 h1:OCs21ST2LrepDfD3lwlQiOqIGp6JiEUqG84GzTDoyJs= @@ -39,7 +33,6 @@ github.com/aws/aws-sdk-go-v2/service/sts v1.26.7/go.mod h1:6h2YuIoxaMSCFf5fi1EgZ github.com/aws/smithy-go v1.19.0 h1:KWFKQV80DpP3vJrrA9sVAHQ5gc2z8i4EzrLhLlWXcBM= github.com/aws/smithy-go v1.19.0/go.mod h1:NukqUGpCZIILqqiV0NIjeFh24kd/FAa4beRb6nbIUPE= github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -47,43 +40,14 @@ github.com/fasthttp/websocket v1.5.7 h1:0a6o2OfeATvtGgoMKleURhLT6JqWPg7fYfWnH4KH github.com/fasthttp/websocket v1.5.7/go.mod h1:bC4fxSono9czeXHQUVKxsC0sNjbm7lPJR04GDFqClfU= github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU= github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA= -github.com/go-openapi/analysis v0.21.2/go.mod h1:HZwRk4RRisyG8vx2Oe6aqeSQcoxRp47Xkp3+K6q+LdY= -github.com/go-openapi/analysis v0.21.4 h1:ZDFLvSNxpDaomuCueM0BlSXxpANBlFYiBvr+GXrvIHc= -github.com/go-openapi/analysis v0.21.4/go.mod h1:4zQ35W4neeZTqh3ol0rv/O8JBbka9QyAgQRPp9y3pfo= -github.com/go-openapi/errors v0.19.8/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= -github.com/go-openapi/errors v0.19.9/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= -github.com/go-openapi/errors v0.20.2/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= -github.com/go-openapi/errors v0.20.3 h1:rz6kiC84sqNQoqrtulzaL/VERgkoCyB6WdEkc2ujzUc= -github.com/go-openapi/errors v0.20.3/go.mod h1:Z3FlZ4I8jEGxjUK+bugx3on2mIAk4txuAOhlsB1FSgk= -github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonpointer v0.20.2 h1:mQc3nmndL8ZBzStEo3JYF8wzmeWffDH4VbXz58sAx6Q= github.com/go-openapi/jsonpointer v0.20.2/go.mod h1:bHen+N0u1KEO3YlmqOjTT9Adn1RfD91Ar825/PuiRVs= -github.com/go-openapi/jsonreference v0.19.6/go.mod h1:diGHMEHg2IqXZGKxqyvWdfWU/aim5Dprw5bqpKkTvns= -github.com/go-openapi/jsonreference v0.20.0/go.mod h1:Ag74Ico3lPc+zR+qjn4XBUmXymS4zJbYVCZmcgkasdo= github.com/go-openapi/jsonreference v0.20.4 h1:bKlDxQxQJgwpUSgOENiMPzCTBVuc7vTdXSSgNeAhojU= github.com/go-openapi/jsonreference v0.20.4/go.mod h1:5pZJyJP2MnYCpoeoMAql78cCHauHj0V9Lhc506VOpw4= -github.com/go-openapi/loads v0.21.1/go.mod h1:/DtAMXXneXFjbQMGEtbamCZb+4x7eGwkvZCvBmwUG+g= -github.com/go-openapi/loads v0.21.2 h1:r2a/xFIYeZ4Qd2TnGpWDIQNcP80dIaZgf704za8enro= -github.com/go-openapi/loads v0.21.2/go.mod h1:Jq58Os6SSGz0rzh62ptiu8Z31I+OTHqmULx5e/gJbNw= -github.com/go-openapi/runtime v0.26.0 h1:HYOFtG00FM1UvqrcxbEJg/SwvDRvYLQKGhw2zaQjTcc= -github.com/go-openapi/runtime v0.26.0/go.mod h1:QgRGeZwrUcSHdeh4Ka9Glvo0ug1LC5WyE+EV88plZrQ= -github.com/go-openapi/spec v0.20.4/go.mod h1:faYFR1CvsJZ0mNsmsphTMSoRrNV3TEDoAM7FOEWeq8I= -github.com/go-openapi/spec v0.20.6/go.mod h1:2OpW+JddWPrpXSCIX8eOx7lZ5iyuWj3RYR6VaaBKcWA= github.com/go-openapi/spec v0.20.13 h1:XJDIN+dLH6vqXgafnl5SUIMnzaChQ6QTo0/UPMbkIaE= github.com/go-openapi/spec v0.20.13/go.mod h1:8EOhTpBoFiask8rrgwbLC3zmJfz4zsCUueRuPM6GNkw= -github.com/go-openapi/strfmt v0.21.0/go.mod h1:ZRQ409bWMj+SOgXofQAGTIo2Ebu72Gs+WaRADcS5iNg= -github.com/go-openapi/strfmt v0.21.1/go.mod h1:I/XVKeLc5+MM5oPNN7P6urMOpuLXEcNrCX/rPGuWb0k= -github.com/go-openapi/strfmt v0.21.3/go.mod h1:k+RzNO0Da+k3FrrynSNN8F7n/peCmQQqbbXjtDfvmGg= -github.com/go-openapi/strfmt v0.21.7 h1:rspiXgNWgeUzhjo1YU01do6qsahtJNByjLVbPLNHb8k= -github.com/go-openapi/strfmt v0.21.7/go.mod h1:adeGTkxE44sPyLk0JV235VQAO/ZXUr8KAzYjclFs3ew= -github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= -github.com/go-openapi/swag v0.21.1/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= github.com/go-openapi/swag v0.22.7 h1:JWrc1uc/P9cSomxfnsFSVWoE1FW6bNbrVPmpQYpCcR8= github.com/go-openapi/swag v0.22.7/go.mod h1:Gl91UqO+btAM0plGGxHqJcQZ1ZTy6jbmridBTsDy8A0= -github.com/go-openapi/validate v0.22.1 h1:G+c2ub6q47kfX1sOBLwIQwzBVt8qmOAARyo/9Fqs9NU= -github.com/go-openapi/validate v0.22.1/go.mod h1:rjnrwK57VJ7A8xqfpAOEKRH8yQSGUriMu5/zuPSQ1hg= github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= @@ -92,77 +56,34 @@ github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJn github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= github.com/go-playground/validator/v10 v10.17.0 h1:SmVVlfAOtlZncTxRuinDPomC2DkXJ4E5T9gDA0AIH74= github.com/go-playground/validator/v10 v10.17.0/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU= -github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0= -github.com/gobuffalo/depgen v0.0.0-20190329151759-d478694a28d3/go.mod h1:3STtPUQYuzV0gBVOY3vy6CfMm/ljR4pABfrTeHNLHUY= -github.com/gobuffalo/depgen v0.1.0/go.mod h1:+ifsuy7fhi15RWncXQQKjWS9JPkdah5sZvtHc2RXGlg= -github.com/gobuffalo/envy v1.6.15/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= -github.com/gobuffalo/envy v1.7.0/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= -github.com/gobuffalo/flect v0.1.0/go.mod h1:d2ehjJqGOH/Kjqcoz+F7jHTBbmDb38yXA598Hb50EGs= -github.com/gobuffalo/flect v0.1.1/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= -github.com/gobuffalo/flect v0.1.3/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= -github.com/gobuffalo/genny v0.0.0-20190329151137-27723ad26ef9/go.mod h1:rWs4Z12d1Zbf19rlsn0nurr75KqhYp52EAGGxTbBhNk= -github.com/gobuffalo/genny v0.0.0-20190403191548-3ca520ef0d9e/go.mod h1:80lIj3kVJWwOrXWWMRzzdhW3DsrdjILVil/SFKBzF28= -github.com/gobuffalo/genny v0.1.0/go.mod h1:XidbUqzak3lHdS//TPu2OgiFB+51Ur5f7CSnXZ/JDvo= -github.com/gobuffalo/genny v0.1.1/go.mod h1:5TExbEyY48pfunL4QSXxlDOmdsD44RRq4mVZ0Ex28Xk= -github.com/gobuffalo/gitgen v0.0.0-20190315122116-cc086187d211/go.mod h1:vEHJk/E9DmhejeLeNt7UVvlSGv3ziL+djtTr3yyzcOw= -github.com/gobuffalo/gogen v0.0.0-20190315121717-8f38393713f5/go.mod h1:V9QVDIxsgKNZs6L2IYiGR8datgMhB577vzTDqypH360= -github.com/gobuffalo/gogen v0.1.0/go.mod h1:8NTelM5qd8RZ15VjQTFkAW6qOMx5wBbW4dSCS3BY8gg= -github.com/gobuffalo/gogen v0.1.1/go.mod h1:y8iBtmHmGc4qa3urIyo1shvOD8JftTtfcKi+71xfDNE= -github.com/gobuffalo/logger v0.0.0-20190315122211-86e12af44bc2/go.mod h1:QdxcLw541hSGtBnhUc4gaNIXRjiDppFGaDqzbrBd3v8= -github.com/gobuffalo/mapi v1.0.1/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= -github.com/gobuffalo/mapi v1.0.2/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= -github.com/gobuffalo/packd v0.0.0-20190315124812-a385830c7fc0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4= -github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4= -github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ= -github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0= -github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw= github.com/gofiber/contrib/fiberzap/v2 v2.1.2 h1:7Z1BqS1sYK9e9jTwqPcWx9qQt46PI8oeswgAp6YNZC4= github.com/gofiber/contrib/fiberzap/v2 v2.1.2/go.mod h1:ulCCQOdDYABGsOQfbndASmCsCN86hsC96iKoOTNYfy8= -github.com/gofiber/contrib/swagger v1.1.1 h1:on+D2fbXkvm0H0lur1rx69mpxLdX1wIH/FrTRZ99b9Y= -github.com/gofiber/contrib/swagger v1.1.1/go.mod h1:pa9awsFSz/3BbSnyTe/drNZaiFfnhC4hk3m9BVet7Co= github.com/gofiber/contrib/websocket v1.3.0 h1:XADFAGorer1VJ1bqC4UkCjqS37kwRTV0415+050NrMk= github.com/gofiber/contrib/websocket v1.3.0/go.mod h1:xguaOzn2ZZ759LavtosEP+rcxIgBEE/rdumPINhR+Xo= github.com/gofiber/fiber/v2 v2.52.2 h1:b0rYH6b06Df+4NyrbdptQL8ifuxw/Tf2DgfkZkDaxEo= github.com/gofiber/fiber/v2 v2.52.2/go.mod h1:KEOE+cXMhXG0zHc9d8+E38hoX+ZN7bhOtgeF2oT6jrQ= -github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/gofiber/swagger v1.0.0 h1:BzUzDS9ZT6fDUa692kxmfOjc1DZiloLiPK/W5z1H1tc= +github.com/gofiber/swagger v1.0.0/go.mod h1:QrYNF1Yrc7ggGK6ATsJ6yfH/8Zi5bu9lA7wB8TmCecg= github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= -github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4= -github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA= -github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.17.6 h1:60eq2E/jlfwQXtvZEeBUYADs+BwKBWURIY+Gj2eRGjI= github.com/klauspost/compress v1.17.6/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= -github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q= github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4= -github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE= -github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= @@ -170,18 +91,6 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= -github.com/mitchellh/mapstructure v1.3.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= -github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= -github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= -github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= -github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= -github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/r3labs/sse/v2 v2.10.0 h1:hFEkLLFY4LDifoHdiCN/LlGBAdVJYsANaLqNYa1l/v0= @@ -189,130 +98,63 @@ github.com/r3labs/sse/v2 v2.10.0/go.mod h1:Igau6Whc+F17QUgML1fYe1VPZzTV6EMCnYktE github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= -github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/savsgio/gotils v0.0.0-20230208104028-c358bd845dee h1:8Iv5m6xEo1NR1AvpV+7XmhI4r39LGNzwUL4YpMuL5vk= github.com/savsgio/gotils v0.0.0-20230208104028-c358bd845dee/go.mod h1:qwtSXrKuJh/zsFQ12yEE89xfCrGKK63Rr7ctU/uCo4g= -github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= -github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= -github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/swaggo/swag v1.16.2 h1:28Pp+8DkQoV+HLzLx8RGJZXNGKbFqnuvSbAAtoxiY04= -github.com/swaggo/swag v1.16.2/go.mod h1:6YzXnDcpr0767iOejs318CwYkCQqyGer6BizOg03f+E= -github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= -github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= -github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/swaggo/files/v2 v2.0.0 h1:hmAt8Dkynw7Ssz46F6pn8ok6YmGZqHSVLZ+HQM7i0kw= +github.com/swaggo/files/v2 v2.0.0/go.mod h1:24kk2Y9NYEJ5lHuCra6iVwkMjIekMCaFq/0JQj66kyM= +github.com/swaggo/swag v1.16.3 h1:PnCYjPCah8FK4I26l2F/KQ4yz3sILcVUN3cTlBFA9Pg= +github.com/swaggo/swag v1.16.3/go.mod h1:DImHIuOFXKpMFAQjcC7FG4m3Dg4+QuUgUzJmKjI/gRk= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasthttp v1.52.0 h1:wqBQpxH71XW0e2g+Og4dzQM8pk34aFYlA1Ga8db7gU0= github.com/valyala/fasthttp v1.52.0/go.mod h1:hf5C4QnVMkNXMspnsUlfM3WitlgYflyhHYoKol/szxQ= github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8= github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= -github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= -github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs= -github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g= -github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM= -github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8= -github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= -go.mongodb.org/mongo-driver v1.7.3/go.mod h1:NqaYOwnXWr5Pm7AOpO5QFxKJ503nbMse/R79oO62zWg= -go.mongodb.org/mongo-driver v1.7.5/go.mod h1:VXEWRZ6URJIkUq2SCAyapmhH0ZLRBP+FT4xhp5Zvxng= -go.mongodb.org/mongo-driver v1.10.0/go.mod h1:wsihk0Kdgv8Kqu1Anit4sfK+22vSFbUrAVEYRhCXrA8= -go.mongodb.org/mongo-driver v1.11.3 h1:Ql6K6qYHEzB6xvu4+AU0BoRoqf9vFPcc4o7MUIdPW8Y= -go.mongodb.org/mongo-driver v1.11.3/go.mod h1:PTSz5yu21bkT/wXpkS7WR5f0ddqw5quethTUn9WM+2g= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= -golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190422162423-af44ce270edf/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= -golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0= golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20191116160921-f9c825593386/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM= -golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190419153524-e8e3143a4f4a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190329151228-23e29df326fe/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190416151739-9c9e1878f421/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190420181800-aa740d480789/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.16.1 h1:TLyB3WofjdOEepBHAU20JdNC1Zbg87elYofWYAY5oZA= golang.org/x/tools v0.16.1/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/cenkalti/backoff.v1 v1.1.0 h1:Arh75ttbsvlpVA7WtVpH4u9h6Zl46xuptxqLxPiSo4Y= gopkg.in/cenkalti/backoff.v1 v1.1.0/go.mod h1:J6Vskwqd+OMVJl8C33mmtxTBs2gyzfv7UDAkHu8BrjI= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/api/http/handlers.go b/pkg/api/http/handlers.go index 5e325ad0..f7cbd20a 100644 --- a/pkg/api/http/handlers.go +++ b/pkg/api/http/handlers.go @@ -122,11 +122,11 @@ func LangStreamChatHandler(tel *telemetry.Telemetry, routerManager *routers.Rout wg sync.WaitGroup ) - chunkResultC := make(chan *schemas.ChatStreamResult) + chatStreamC := make(chan *schemas.ChatStreamMessage) router, _ := routerManager.GetLangRouter(routerID) - defer close(chunkResultC) + defer close(chatStreamC) defer c.Conn.Close() wg.Add(1) @@ -134,16 +134,8 @@ func LangStreamChatHandler(tel *telemetry.Telemetry, routerManager *routers.Rout go func() { defer wg.Done() - for chunkResult := range chunkResultC { - if chunkResult.Error() != nil { - if err = c.WriteJSON(chunkResult.Error()); err != nil { - break - } - - continue - } - - if err = c.WriteJSON(chunkResult.Chunk()); err != nil { + for chatStreamMsg := range chatStreamC { + if err = c.WriteJSON(chatStreamMsg); err != nil { break } } @@ -168,7 +160,7 @@ func LangStreamChatHandler(tel *telemetry.Telemetry, routerManager *routers.Rout go func(chatRequest schemas.ChatStreamRequest) { defer wg.Done() - router.ChatStream(context.Background(), &chatRequest, chunkResultC) + router.ChatStream(context.Background(), &chatRequest, chatStreamC) }(chatRequest) } diff --git a/pkg/api/http/server.go b/pkg/api/http/server.go index e49c8fec..094b6c4f 100644 --- a/pkg/api/http/server.go +++ b/pkg/api/http/server.go @@ -6,9 +6,11 @@ import ( "fmt" "time" - "github.com/gofiber/contrib/fiberzap/v2" + "github.com/gofiber/swagger" + + "glide/docs" - "github.com/gofiber/contrib/swagger" + "github.com/gofiber/contrib/fiberzap/v2" "github.com/gofiber/fiber/v2" _ "glide/docs" // importing docs package to include them into the binary @@ -37,12 +39,10 @@ func NewServer(config *ServerConfig, tel *telemetry.Telemetry, routerManager *ro } func (srv *Server) Run() error { - srv.server.Use(swagger.New(swagger.Config{ - Title: "Glide API Docs", - BasePath: "/v1/", - Path: "swagger", - FilePath: "./docs/swagger.yaml", - })) + // TODO: refactor this when https://github.com/gofiber/contrib/pull/1069 is merged + srv.server.Get("/swagger.json", func(c *fiber.Ctx) error { + return c.Status(fiber.StatusOK).Type("json").Send(docs.SwaggerJSON) + }) srv.server.Use(fiberzap.New(fiberzap.Config{ Logger: srv.telemetry.Logger, @@ -50,6 +50,11 @@ func (srv *Server) Run() error { v1 := srv.server.Group("/v1") + v1.Get("/swagger/*", swagger.New(swagger.Config{ + Title: "Glide API Docs", + URL: "/swagger.json", + })) + v1.Get("/language/", LangRoutersHandler(srv.routerManager)) v1.Post("/language/:router/chat/", LangChatHandler(srv.routerManager)) diff --git a/pkg/api/schemas/chat_stream.go b/pkg/api/schemas/chat_stream.go index d77310c7..983d2242 100644 --- a/pkg/api/schemas/chat_stream.go +++ b/pkg/api/schemas/chat_stream.go @@ -1,15 +1,34 @@ package schemas +import "time" + type ( Metadata = map[string]any + EventType = string FinishReason = string + ErrorCode = string +) + +var ( + Complete FinishReason = "complete" + MaxTokens FinishReason = "max_tokens" + ContentFiltered FinishReason = "content_filtered" + ErrorReason FinishReason = "error" + OtherReason FinishReason = "other" +) + +var ( + NoModelConfigured ErrorCode = "no_model_configured" + ModelUnavailable ErrorCode = "model_unavailable" + AllModelsUnavailable ErrorCode = "all_models_unavailable" + UnknownError ErrorCode = "unknown_error" ) -var Complete FinishReason = "complete" +type StreamRequestID = string // ChatStreamRequest defines a message that requests a new streaming chat type ChatStreamRequest struct { - ID string `json:"id" validate:"required"` + ID StreamRequestID `json:"id" validate:"required"` Message ChatMessage `json:"message" validate:"required"` MessageHistory []ChatMessage `json:"messageHistory" validate:"required"` Override *OverrideChatRequest `json:"overrideMessage,omitempty"` @@ -27,54 +46,67 @@ func NewChatStreamFromStr(message string) *ChatStreamRequest { } type ModelChunkResponse struct { - Metadata *Metadata `json:"metadata,omitempty"` - Message ChatMessage `json:"message"` - FinishReason *FinishReason `json:"finishReason,omitempty"` + Metadata *Metadata `json:"metadata,omitempty"` + Message ChatMessage `json:"message"` +} + +type ChatStreamMessage struct { + ID StreamRequestID `json:"id"` + CreatedAt int `json:"createdAt"` + RouterID string `json:"routerId"` + Metadata *Metadata `json:"metadata,omitempty"` + Chunk *ChatStreamChunk `json:"chunk,omitempty"` + Error *ChatStreamError `json:"error,omitempty"` } // ChatStreamChunk defines a message for a chunk of streaming chat response type ChatStreamChunk struct { - ID string `json:"id"` - CreatedAt int `json:"createdAt"` - Provider string `json:"providerId"` - RouterID string `json:"routerId"` ModelID string `json:"modelId"` - Cached bool `json:"cached"` + Provider string `json:"providerName"` ModelName string `json:"modelName"` - Metadata *Metadata `json:"metadata,omitempty"` + Cached bool `json:"cached"` ModelResponse ModelChunkResponse `json:"modelResponse"` + FinishReason *FinishReason `json:"finishReason,omitempty"` } type ChatStreamError struct { - ID string `json:"id"` - ErrCode string `json:"errCode"` - Message string `json:"message"` - Metadata *Metadata `json:"metadata,omitempty"` -} - -type ChatStreamResult struct { - chunk *ChatStreamChunk - err *ChatStreamError -} - -func (r *ChatStreamResult) Chunk() *ChatStreamChunk { - return r.chunk -} - -func (r *ChatStreamResult) Error() *ChatStreamError { - return r.err + ErrCode ErrorCode `json:"errCode"` + Message string `json:"message"` + FinishReason *FinishReason `json:"finishReason,omitempty"` } -func NewChatStreamResult(chunk *ChatStreamChunk) *ChatStreamResult { - return &ChatStreamResult{ - chunk: chunk, - err: nil, +func NewChatStreamChunk( + reqID StreamRequestID, + routerID string, + reqMetadata *Metadata, + chunk *ChatStreamChunk, +) *ChatStreamMessage { + return &ChatStreamMessage{ + ID: reqID, + RouterID: routerID, + CreatedAt: int(time.Now().UTC().Unix()), + Metadata: reqMetadata, + Chunk: chunk, } } -func NewChatStreamErrorResult(err *ChatStreamError) *ChatStreamResult { - return &ChatStreamResult{ - chunk: nil, - err: err, +func NewChatStreamError( + reqID StreamRequestID, + routerID string, + errCode ErrorCode, + errMsg string, + reqMetadata *Metadata, + finishReason *FinishReason, +) *ChatStreamMessage { + return &ChatStreamMessage{ + ID: reqID, + RouterID: routerID, + CreatedAt: int(time.Now().UTC().Unix()), + Metadata: reqMetadata, + Error: &ChatStreamError{ + ErrCode: errCode, + Message: errMsg, + FinishReason: finishReason, + }, } } diff --git a/pkg/config/fields/duration.go b/pkg/config/fields/duration.go new file mode 100644 index 00000000..0ace8464 --- /dev/null +++ b/pkg/config/fields/duration.go @@ -0,0 +1,10 @@ +package fields + +import "time" + +type Duration time.Duration + +// MarshalText serializes Durations in a human-friendly way (it's shown in nanoseconds by default) +func (d Duration) MarshalText() ([]byte, error) { + return []byte(time.Duration(d).String()), nil +} diff --git a/pkg/config/provider.go b/pkg/config/provider.go index 5101aecc..63283dd9 100644 --- a/pkg/config/provider.go +++ b/pkg/config/provider.go @@ -72,6 +72,10 @@ func (p *Provider) Load(configPath string) (*Provider, error) { return p, nil } +func Indent(text string, level int) string { + return strings.Repeat(" ", level) + text +} + func (p *Provider) formatValidationError(configPath string, err error) error { // this check is only needed when your code could produce // an invalid value for validation such as interface with nil @@ -86,14 +90,14 @@ func (p *Provider) formatValidationError(configPath string, err error) error { errors = append( errors, fmt.Sprintf( - "- ❌ %v", p.formatFieldError(fieldErr), + Indent("βœ— %v", 1), p.formatFieldError(fieldErr), ), ) } // from here you can create your own error messages in whatever language you wish return fmt.Errorf( - "invalid config file %v:\n%v\nPlease make sure the config file is properly formatted", + "invalid config file %v:\n\n%v\n\nPlease make sure the config file is properly formatted", configPath, strings.Join(errors, "\n"), ) @@ -105,19 +109,27 @@ func (p *Provider) formatFieldError(fieldErr validator.FieldError) string { switch fieldErr.Tag() { case "required": return fmt.Sprintf( - "\"%v\"field is required, \"%v\" provided", + "%v field is required, \"%v\" provided", namespace, fieldErr.Value(), ) case "min": if fieldErr.Kind() == reflect.Map || fieldErr.Kind() == reflect.Slice { - return fmt.Sprintf("\"%v\" field must have at least %s element(s)", namespace, fieldErr.Param()) + return fmt.Sprintf("%v field must have at least %s element(s)", namespace, fieldErr.Param()) + } + + return fmt.Sprintf("%v field must have minimum value: %q", namespace, fieldErr.Param()) + case "max": + if fieldErr.Kind() == reflect.Map || fieldErr.Kind() == reflect.Slice { + return fmt.Sprintf("%v field must have at most %s element(s)", namespace, fieldErr.Param()) } - return fmt.Sprintf("\"%v\" field must have minimum value: %q", namespace, fieldErr.Param()) + return fmt.Sprintf("%v field must have maximum value: %q", namespace, fieldErr.Param()) + case "lte": + return fmt.Sprintf("%v field must less than or equal to: %q", namespace, fieldErr.Param()) default: return fmt.Sprintf( - "\"%v\"field: %v", + "%v field: %v", namespace, fieldErr.Tag(), ) diff --git a/pkg/gateway.go b/pkg/gateway.go index 6d2a6092..c372482a 100644 --- a/pkg/gateway.go +++ b/pkg/gateway.go @@ -44,7 +44,7 @@ func NewGateway(configProvider *config.Provider) (*Gateway, error) { } tel.L().Info("🐦Glide is starting up", zap.String("version", version.FullVersion)) - tel.L().Debug("βœ… config loaded successfully:\n" + configProvider.GetStr()) + tel.L().Debug("βœ… Config loaded successfully:\n" + configProvider.GetStr()) routerManager, err := routers.NewManager(&cfg.Routers, tel) if err != nil { diff --git a/pkg/providers/azureopenai/chat_stream.go b/pkg/providers/azureopenai/chat_stream.go index a3a64bff..6d44345d 100644 --- a/pkg/providers/azureopenai/chat_stream.go +++ b/pkg/providers/azureopenai/chat_stream.go @@ -10,6 +10,7 @@ import ( "github.com/r3labs/sse/v2" "glide/pkg/providers/clients" + "glide/pkg/providers/openai" "glide/pkg/telemetry" "go.uber.org/zap" @@ -17,38 +18,32 @@ import ( "glide/pkg/api/schemas" ) -var ( - StopReason = "stop" - streamDoneMarker = []byte("[DONE]") -) +// TODO: Think about reducing the number of copy-pasted code btw OpenAI and Azure providers // ChatStream represents chat stream for a specific request type ChatStream struct { - tel *telemetry.Telemetry - client *http.Client - req *http.Request - reqID string - reqMetadata *schemas.Metadata - resp *http.Response - reader *sse.EventStreamReader - errMapper *ErrorMapper + tel *telemetry.Telemetry + client *http.Client + req *http.Request + resp *http.Response + reader *sse.EventStreamReader + finishReasonMapper *openai.FinishReasonMapper + errMapper *ErrorMapper } func NewChatStream( tel *telemetry.Telemetry, client *http.Client, req *http.Request, - reqID string, - reqMetadata *schemas.Metadata, + finishReasonMapper *openai.FinishReasonMapper, errMapper *ErrorMapper, ) *ChatStream { return &ChatStream{ - tel: tel, - client: client, - req: req, - reqID: reqID, - reqMetadata: reqMetadata, - errMapper: errMapper, + tel: tel, + client: client, + req: req, + finishReasonMapper: finishReasonMapper, + errMapper: errMapper, } } @@ -96,7 +91,7 @@ func (s *ChatStream) Recv() (*schemas.ChatStreamChunk, error) { event, err := clients.ParseSSEvent(rawEvent) - if bytes.Equal(event.Data, streamDoneMarker) { + if bytes.Equal(event.Data, openai.StreamDoneMarker) { s.tel.L().Info( "EOF: [DONE] marker found in chat stream", zap.String("provider", providerName), @@ -126,19 +121,11 @@ func (s *ChatStream) Recv() (*schemas.ChatStreamChunk, error) { responseChunk := completionChunk.Choices[0] - var finishReason *schemas.FinishReason - - if responseChunk.FinishReason == StopReason { - finishReason = &schemas.Complete - } - // TODO: use objectpool here return &schemas.ChatStreamChunk{ - ID: s.reqID, - Provider: providerName, Cached: false, + Provider: providerName, ModelName: completionChunk.ModelName, - Metadata: s.reqMetadata, ModelResponse: schemas.ModelChunkResponse{ Metadata: &schemas.Metadata{ "response_id": completionChunk.ID, @@ -148,8 +135,8 @@ func (s *ChatStream) Recv() (*schemas.ChatStreamChunk, error) { Role: responseChunk.Delta.Role, Content: responseChunk.Delta.Content, }, - FinishReason: finishReason, }, + FinishReason: s.finishReasonMapper.Map(responseChunk.FinishReason), }, nil } } @@ -177,8 +164,7 @@ func (c *Client) ChatStream(ctx context.Context, req *schemas.ChatStreamRequest) c.tel, c.httpClient, httpRequest, - req.ID, - req.Metadata, + c.finishReasonMapper, c.errMapper, ), nil } diff --git a/pkg/providers/azureopenai/client.go b/pkg/providers/azureopenai/client.go index 9a15aeb8..6fa26d88 100644 --- a/pkg/providers/azureopenai/client.go +++ b/pkg/providers/azureopenai/client.go @@ -5,6 +5,8 @@ import ( "fmt" "net/http" + "glide/pkg/providers/openai" + "glide/pkg/providers/clients" "glide/pkg/telemetry" ) @@ -23,6 +25,7 @@ type Client struct { baseURL string // The name of your Azure OpenAI Resource (e.g https://glide-test.openai.azure.com/) chatURL string chatRequestTemplate *ChatRequest + finishReasonMapper *openai.FinishReasonMapper errMapper *ErrorMapper config *Config httpClient *http.Client @@ -43,6 +46,7 @@ func NewClient(providerConfig *Config, clientConfig *clients.ClientConfig, tel * chatURL: chatURL, config: providerConfig, chatRequestTemplate: NewChatRequestFromConfig(providerConfig), + finishReasonMapper: openai.NewFinishReasonMapper(tel), errMapper: NewErrorMapper(tel), httpClient: &http.Client{ // TODO: use values from the config diff --git a/pkg/providers/cohere/chat.go b/pkg/providers/cohere/chat.go index 86573041..beb6403c 100644 --- a/pkg/providers/cohere/chat.go +++ b/pkg/providers/cohere/chat.go @@ -20,13 +20,10 @@ func NewChatRequestFromConfig(cfg *Config) *ChatRequest { return &ChatRequest{ Model: cfg.Model, Temperature: cfg.DefaultParams.Temperature, - PreambleOverride: cfg.DefaultParams.PreambleOverride, - ChatHistory: cfg.DefaultParams.ChatHistory, - ConversationID: cfg.DefaultParams.ConversationID, + Preamble: cfg.DefaultParams.Preamble, PromptTruncation: cfg.DefaultParams.PromptTruncation, Connectors: cfg.DefaultParams.Connectors, SearchQueriesOnly: cfg.DefaultParams.SearchQueriesOnly, - CitiationQuality: cfg.DefaultParams.CitiationQuality, Stream: false, } } @@ -55,15 +52,16 @@ func (c *Client) createRequestSchema(request *schemas.ChatRequest) *ChatRequest // Build the Cohere specific ChatHistory if len(request.MessageHistory) > 0 { - chatRequest.ChatHistory = make([]ChatHistory, len(request.MessageHistory)) - for i, message := range request.MessageHistory { - chatRequest.ChatHistory[i] = ChatHistory{ - // Copy the necessary fields from message to ChatHistory - // For example, if ChatHistory has a field called "Text", you can do: - Role: message.Role, - Message: message.Content, - User: "", - } + chatRequest.ChatHistory = make([]ChatMessage, 0, len(request.MessageHistory)) + + for _, message := range request.MessageHistory { + chatRequest.ChatHistory = append( + chatRequest.ChatHistory, + ChatMessage{ + Role: message.Role, + Content: message.Content, + }, + ) } } @@ -108,12 +106,12 @@ func (c *Client) doChatRequest(ctx context.Context, payload *ChatRequest) (*sche c.tel.Logger.Error( "cohere chat request failed", zap.Int("status_code", resp.StatusCode), - zap.String("response", string(bodyBytes)), + zap.ByteString("response", bodyBytes), zap.Any("headers", resp.Header), ) if resp.StatusCode != http.StatusOK { - return c.handleErrorResponse(resp) + return nil, c.errMapper.Map(resp) } // Server & client errors result in the same error to keep gateway resilient @@ -127,15 +125,6 @@ func (c *Client) doChatRequest(ctx context.Context, payload *ChatRequest) (*sche return nil, err } - // Parse the response JSON - var responseJSON map[string]interface{} - - err = json.Unmarshal(bodyBytes, &responseJSON) - if err != nil { - c.tel.Logger.Error("failed to parse cohere chat response", zap.Error(err)) - return nil, err - } - // Parse the response JSON var cohereCompletion ChatCompletion @@ -172,44 +161,3 @@ func (c *Client) doChatRequest(ctx context.Context, payload *ChatRequest) (*sche return &response, nil } - -func (c *Client) handleErrorResponse(resp *http.Response) (*schemas.ChatResponse, error) { - bodyBytes, err := io.ReadAll(resp.Body) - if err != nil { - c.tel.Logger.Error("failed to read cohere chat response", zap.Error(err)) - return nil, err - } - - c.tel.Logger.Error( - "cohere chat request failed", - zap.Int("status_code", resp.StatusCode), - zap.String("response", string(bodyBytes)), - zap.Any("headers", resp.Header), - ) - - if resp.StatusCode == http.StatusTooManyRequests { - cooldownDelay, err := c.getCooldownDelay(resp) - if err != nil { - return nil, fmt.Errorf("failed to parse cooldown delay from headers: %w", err) - } - - return nil, clients.NewRateLimitError(&cooldownDelay) - } - - if resp.StatusCode == http.StatusUnauthorized { - return nil, clients.ErrUnauthorized - } - - return nil, clients.ErrProviderUnavailable -} - -func (c *Client) getCooldownDelay(resp *http.Response) (time.Duration, error) { - retryAfter := resp.Header.Get("Retry-After") - - cooldownDelay, err := time.ParseDuration(retryAfter) - if err != nil { - return 0, fmt.Errorf("failed to parse cooldown delay from headers: %w", err) - } - - return cooldownDelay, nil -} diff --git a/pkg/providers/cohere/chat_stream.go b/pkg/providers/cohere/chat_stream.go index f6f9e9a8..aeb3cbec 100644 --- a/pkg/providers/cohere/chat_stream.go +++ b/pkg/providers/cohere/chat_stream.go @@ -8,7 +8,6 @@ import ( "io" "net/http" - "github.com/r3labs/sse/v2" "glide/pkg/providers/clients" "glide/pkg/telemetry" @@ -17,35 +16,46 @@ import ( "glide/pkg/api/schemas" ) -var StopReason = "stream-end" +// SupportedEventType Cohere has other types too: +// Ref: https://docs.cohere.com/reference/chat (see Chat -> Responses -> StreamedChatResponse) +type SupportedEventType = string + +var ( + StreamStartEvent SupportedEventType = "stream-start" + TextGenEvent SupportedEventType = "text-generation" + StreamEndEvent SupportedEventType = "stream-end" +) // ChatStream represents cohere chat stream for a specific request type ChatStream struct { - tel *telemetry.Telemetry - client *http.Client - req *http.Request - reqID string - reqMetadata *schemas.Metadata - resp *http.Response - reader *sse.EventStreamReader - errMapper *ErrorMapper + client *http.Client + req *http.Request + modelName string + resp *http.Response + generationID string + streamFinished bool + reader *StreamReader + errMapper *ErrorMapper + finishReasonMapper *FinishReasonMapper + tel *telemetry.Telemetry } func NewChatStream( tel *telemetry.Telemetry, client *http.Client, req *http.Request, - reqID string, - reqMetadata *schemas.Metadata, + modelName string, errMapper *ErrorMapper, + finishReasonMapper *FinishReasonMapper, ) *ChatStream { return &ChatStream{ - tel: tel, - client: client, - req: req, - reqID: reqID, - reqMetadata: reqMetadata, - errMapper: errMapper, + tel: tel, + client: client, + req: req, + modelName: modelName, + errMapper: errMapper, + streamFinished: false, + finishReasonMapper: finishReasonMapper, } } @@ -59,17 +69,23 @@ func (s *ChatStream) Open() error { return s.errMapper.Map(resp) } + s.tel.L().Debug("Resp Headers", zap.Any("headers", resp.Header)) + s.resp = resp - s.reader = sse.NewEventStreamReader(resp.Body, 8192) // TODO: should we expose maxBufferSize? + s.reader = NewStreamReader(resp.Body, 8192) // TODO: should we expose maxBufferSize? return nil } func (s *ChatStream) Recv() (*schemas.ChatStreamChunk, error) { - var completionChunk ChatCompletionChunk + if s.streamFinished { + return nil, io.EOF + } + + var responseChunk ChatCompletionChunk for { - rawEvent, err := s.reader.ReadEvent() + rawChunk, err := s.reader.ReadEvent() if err != nil { s.tel.L().Warn( "Chat stream is unexpectedly disconnected", @@ -77,12 +93,7 @@ func (s *ChatStream) Recv() (*schemas.ChatStreamChunk, error) { zap.Error(err), ) - if err == io.EOF { - return nil, io.EOF - } - - // if err is io.EOF, this still means that the stream is interrupted unexpectedly - // because the normal stream termination is done via finding out streamDoneMarker + // if io.EOF occurred in the middle of the stream, then the stream was interrupted return nil, clients.ErrProviderUnavailable } @@ -90,69 +101,65 @@ func (s *ChatStream) Recv() (*schemas.ChatStreamChunk, error) { s.tel.L().Debug( "Raw chat stream chunk", zap.String("provider", providerName), - zap.ByteString("rawChunk", rawEvent), + zap.ByteString("rawChunk", rawChunk), ) - event, err := clients.ParseSSEvent(rawEvent) + err = json.Unmarshal(rawChunk, &responseChunk) if err != nil { - return nil, fmt.Errorf("failed to parse chat stream message: %v", err) + return nil, fmt.Errorf("failed to unmarshal chat stream chunk: %v", err) } - if !event.HasContent() { + if responseChunk.EventType == StreamStartEvent { + s.generationID = *responseChunk.GenerationID + + continue + } + + if responseChunk.EventType != TextGenEvent && responseChunk.EventType != StreamEndEvent { s.tel.L().Debug( - "Received an empty message in chat stream, skipping it", + "Unsupported stream chunk type, skipping it", zap.String("provider", providerName), - zap.Any("msg", event), + zap.ByteString("chunk", rawChunk), ) continue } - err = json.Unmarshal(event.Data, &completionChunk) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal chat stream chunk: %v", err) - } - - responseChunk := completionChunk - - var finishReason *schemas.FinishReason - if responseChunk.IsFinished { - finishReason = &schemas.Complete + s.streamFinished = true + // TODO: use objectpool here return &schemas.ChatStreamChunk{ - ID: s.reqID, - Provider: providerName, Cached: false, - ModelName: "NA", - Metadata: s.reqMetadata, + Provider: providerName, + ModelName: s.modelName, ModelResponse: schemas.ModelChunkResponse{ Metadata: &schemas.Metadata{ - "generationId": responseChunk.Response.GenerationID, - "responseId": responseChunk.Response.ResponseID, + "generation_id": s.generationID, + "response_id": responseChunk.Response.ResponseID, }, Message: schemas.ChatMessage{ Role: "model", Content: responseChunk.Text, }, - FinishReason: finishReason, }, + FinishReason: s.finishReasonMapper.Map(responseChunk.FinishReason), }, nil } // TODO: use objectpool here return &schemas.ChatStreamChunk{ - ID: s.reqID, - Provider: providerName, Cached: false, - ModelName: "NA", - Metadata: s.reqMetadata, + Provider: providerName, + ModelName: s.modelName, ModelResponse: schemas.ModelChunkResponse{ + Metadata: &schemas.Metadata{ + "generation_id": s.generationID, + }, Message: schemas.ChatMessage{ Role: "model", Content: responseChunk.Text, }, - FinishReason: finishReason, }, }, nil } @@ -181,29 +188,29 @@ func (c *Client) ChatStream(ctx context.Context, req *schemas.ChatStreamRequest) c.tel, c.httpClient, httpRequest, - req.ID, - req.Metadata, + c.chatRequestTemplate.Model, c.errMapper, + c.finishReasonMapper, ), nil } func (c *Client) createRequestFromStream(request *schemas.ChatStreamRequest) *ChatRequest { // TODO: consider using objectpool to optimize memory allocation chatRequest := *c.chatRequestTemplate // hoping to get a copy of the template - chatRequest.Message = request.Message.Content // Build the Cohere specific ChatHistory if len(request.MessageHistory) > 0 { - chatRequest.ChatHistory = make([]ChatHistory, len(request.MessageHistory)) - for i, message := range request.MessageHistory { - chatRequest.ChatHistory[i] = ChatHistory{ - // Copy the necessary fields from message to ChatHistory - // For example, if ChatHistory has a field called "Text", you can do: - Role: message.Role, - Message: message.Content, - User: "", - } + chatRequest.ChatHistory = make([]ChatMessage, 0, len(request.MessageHistory)) + + for _, message := range request.MessageHistory { + chatRequest.ChatHistory = append( + chatRequest.ChatHistory, + ChatMessage{ + Role: message.Role, + Content: message.Content, + }, + ) } } diff --git a/pkg/providers/cohere/chat_stream_test.go b/pkg/providers/cohere/chat_stream_test.go index 3552b45b..c89e646c 100644 --- a/pkg/providers/cohere/chat_stream_test.go +++ b/pkg/providers/cohere/chat_stream_test.go @@ -50,7 +50,7 @@ func TestCohere_ChatStreamRequest(t *testing.T) { t.Errorf("error reading cohere chat mock response: %v", err) } - w.Header().Set("Content-Type", "text/event-stream") + w.Header().Set("Content-Type", "application/stream+json") _, err = w.Write(chatResponse) if err != nil { @@ -94,7 +94,7 @@ func TestCohere_ChatStreamRequest(t *testing.T) { func TestCohere_ChatStreamRequestInterrupted(t *testing.T) { tests := map[string]string{ - "success stream, but with empty event": "./testdata/chat_stream.empty.txt", + "interrupted stream": "./testdata/chat_stream.interrupted.txt", } for name, streamFile := range tests { @@ -141,16 +141,17 @@ func TestCohere_ChatStreamRequestInterrupted(t *testing.T) { err = stream.Open() require.NoError(t, err) - for { + for range 5 { chunk, err := stream.Recv() - if err != nil { - require.ErrorIs(t, err, io.EOF) - return - } require.NoError(t, err) require.NotNil(t, chunk) } + + chunk, err := stream.Recv() + + require.Error(t, err) + require.Nil(t, chunk) }) } } diff --git a/pkg/providers/cohere/client.go b/pkg/providers/cohere/client.go index ec778c15..b266387a 100644 --- a/pkg/providers/cohere/client.go +++ b/pkg/providers/cohere/client.go @@ -23,6 +23,7 @@ type Client struct { baseURL string chatURL string chatRequestTemplate *ChatRequest + finishReasonMapper *FinishReasonMapper errMapper *ErrorMapper config *Config httpClient *http.Client @@ -49,7 +50,9 @@ func NewClient(providerConfig *Config, clientConfig *clients.ClientConfig, tel * MaxIdleConnsPerHost: 2, }, }, - tel: tel, + errMapper: NewErrorMapper(tel), + finishReasonMapper: NewFinishReasonMapper(tel), + tel: tel, } return c, nil diff --git a/pkg/providers/cohere/config.go b/pkg/providers/cohere/config.go index 50dbcde4..3688b7c1 100644 --- a/pkg/providers/cohere/config.go +++ b/pkg/providers/cohere/config.go @@ -7,28 +7,26 @@ import ( // Params defines Cohere-specific model params with the specific validation of values // TODO: Add validations type Params struct { - Temperature float64 `json:"temperature,omitempty"` - Stream bool `json:"stream,omitempty"` - PreambleOverride string `json:"preamble_override,omitempty"` - ChatHistory []ChatHistory `json:"chat_history,omitempty"` - ConversationID string `json:"conversation_id,omitempty"` - PromptTruncation string `json:"prompt_truncation,omitempty"` - Connectors []string `json:"connectors,omitempty"` - SearchQueriesOnly bool `json:"search_queries_only,omitempty"` - CitiationQuality string `json:"citiation_quality,omitempty"` + Seed *int `yaml:"seed,omitempty" json:"seed,omitempty" validate:"omitempty,number"` + Temperature float64 `yaml:"temperature,omitempty" json:"temperature" validate:"required,number"` + MaxTokens *int `yaml:"max_tokens,omitempty" json:"max_tokens,omitempty" validate:"omitempty,number"` + K int `yaml:"k,omitempty" json:"k" validate:"number,gte=0,lte=500"` + P float32 `yaml:"p,omitempty" json:"p" validate:"number,gte=0.01,lte=0.99"` + FrequencyPenalty float32 `yaml:"frequency_penalty,omitempty" json:"frequency_penalty" validate:"gte=0.0,lte=1.0"` + PresencePenalty float32 `yaml:"presence_penalty,omitempty" json:"presence_penalty" validate:"gte=0.0,lte=1.0"` + Preamble string `yaml:"preamble,omitempty" json:"preamble,omitempty"` + StopSequences []string `yaml:"stop_sequences,omitempty" json:"stop_sequences" validate:"max=5"` + PromptTruncation *string `yaml:"prompt_truncation,omitempty" json:"prompt_truncation,omitempty"` + Connectors []string `yaml:"connectors,omitempty" json:"connectors,omitempty"` + SearchQueriesOnly bool `yaml:"search_queries_only,omitempty" json:"search_queries_only,omitempty"` } func DefaultParams() Params { return Params{ Temperature: 0.3, - Stream: false, - PreambleOverride: "", - ChatHistory: nil, - ConversationID: "", - PromptTruncation: "", - Connectors: []string{}, + K: 0, + P: .75, SearchQueriesOnly: false, - CitiationQuality: "", } } @@ -41,9 +39,9 @@ func (p *Params) UnmarshalYAML(unmarshal func(interface{}) error) error { } type Config struct { - BaseURL string `yaml:"base_url" json:"baseUrl" validate:"required"` + BaseURL string `yaml:"base_url" json:"baseUrl" validate:"required,http_url"` ChatEndpoint string `yaml:"chat_endpoint" json:"chatEndpoint" validate:"required"` - Model string `yaml:"model" json:"model" validate:"required"` + Model string `yaml:"model" json:"model" validate:"required"` // https://docs.cohere.com/docs/models#command APIKey fields.Secret `yaml:"api_key" json:"-" validate:"required"` DefaultParams *Params `yaml:"default_params,omitempty" json:"defaultParams"` } diff --git a/pkg/providers/cohere/error.go b/pkg/providers/cohere/errors.go similarity index 100% rename from pkg/providers/cohere/error.go rename to pkg/providers/cohere/errors.go diff --git a/pkg/providers/cohere/finish_reason.go b/pkg/providers/cohere/finish_reason.go new file mode 100644 index 00000000..6133ea39 --- /dev/null +++ b/pkg/providers/cohere/finish_reason.go @@ -0,0 +1,53 @@ +package cohere + +import ( + "strings" + + "glide/pkg/api/schemas" + "glide/pkg/telemetry" + "go.uber.org/zap" +) + +var ( + // Reference: https://platform.openai.com/docs/api-reference/chat/object + CompleteReason = "complete" + MaxTokensReason = "max_tokens" + FilteredReason = "error_toxic" + // TODO: How to process ERROR_LIMIT & ERROR? +) + +func NewFinishReasonMapper(tel *telemetry.Telemetry) *FinishReasonMapper { + return &FinishReasonMapper{ + tel: tel, + } +} + +type FinishReasonMapper struct { + tel *telemetry.Telemetry +} + +func (m *FinishReasonMapper) Map(finishReason *string) *schemas.FinishReason { + if finishReason == nil || len(*finishReason) == 0 { + return nil + } + + var reason *schemas.FinishReason + + switch strings.ToLower(*finishReason) { + case CompleteReason: + reason = &schemas.Complete + case MaxTokensReason: + reason = &schemas.MaxTokens + case FilteredReason: + reason = &schemas.ContentFiltered + default: + m.tel.Logger.Warn( + "Unknown finish reason, other is going to used", + zap.String("unknown_reason", *finishReason), + ) + + reason = &schemas.OtherReason + } + + return reason +} diff --git a/pkg/providers/cohere/schemas.go b/pkg/providers/cohere/schemas.go index f6fc310e..8516b306 100644 --- a/pkg/providers/cohere/schemas.go +++ b/pkg/providers/cohere/schemas.go @@ -69,10 +69,12 @@ type ConnectorsResponse struct { // ChatCompletionChunk represents SSEvent a chat response is broken down on chat streaming // Ref: https://docs.cohere.com/reference/about type ChatCompletionChunk struct { - IsFinished bool `json:"is_finished"` - EventType string `json:"event_type"` - Text string `json:"text"` - Response FinalResponse `json:"response,omitempty"` + IsFinished bool `json:"is_finished"` + EventType string `json:"event_type"` + GenerationID *string `json:"generation_id"` + Text string `json:"text"` + Response *FinalResponse `json:"response,omitempty"` + FinishReason *string `json:"finish_reason,omitempty"` } type FinalResponse struct { @@ -81,33 +83,32 @@ type FinalResponse struct { GenerationID string `json:"generation_id"` TokenCount TokenCount `json:"token_count"` Meta Meta `json:"meta"` - FinishReason string `json:"finish_reason"` } type ChatMessage struct { - Role string `json:"role"` + Role string `json:"role"` // CHATBOT, SYSTEM, USER Content string `json:"content"` } -type ChatHistory struct { - Role string `json:"role"` - Message string `json:"message"` - User string `json:"user,omitempty"` -} - -// ChatRequest is a request to complete a chat completion.. +// ChatRequest is a request to complete a chat completion +// Ref: https://docs.cohere.com/reference/chat type ChatRequest struct { Model string `json:"model"` Message string `json:"message"` + ChatHistory []ChatMessage `json:"chat_history"` Temperature float64 `json:"temperature,omitempty"` - PreambleOverride string `json:"preamble_override,omitempty"` - ChatHistory []ChatHistory `json:"chat_history,omitempty"` - ConversationID string `json:"conversation_id,omitempty"` - PromptTruncation string `json:"prompt_truncation,omitempty"` + Preamble string `json:"preamble,omitempty"` + PromptTruncation *string `json:"prompt_truncation,omitempty"` Connectors []string `json:"connectors,omitempty"` SearchQueriesOnly bool `json:"search_queries_only,omitempty"` - CitiationQuality string `json:"citiation_quality,omitempty"` Stream bool `json:"stream,omitempty"` + Seed *int `json:"seed,omitempty"` + MaxTokens *int `json:"max_tokens,omitempty"` + K int `json:"k"` + P float32 `json:"p"` + FrequencyPenalty float32 `json:"frequency_penalty"` + PresencePenalty float32 `json:"presence_penalty"` + StopSequences []string `json:"stop_sequences"` } type Connectors struct { diff --git a/pkg/providers/cohere/stream_reader.go b/pkg/providers/cohere/stream_reader.go new file mode 100644 index 00000000..259f17bc --- /dev/null +++ b/pkg/providers/cohere/stream_reader.go @@ -0,0 +1,73 @@ +package cohere + +import ( + "bufio" + "bytes" + "context" + "io" +) + +// StreamReader reads Cohere streaming chat chunks that are formated +// as serializer chunk json per line (a.k.a. application/stream+json) +type StreamReader struct { + scanner *bufio.Scanner +} + +func containNewline(data []byte) (int, int) { + return bytes.Index(data, []byte("\n")), 1 +} + +// NewStreamReader creates an instance of StreamReader +func NewStreamReader(stream io.Reader, maxBufferSize int) *StreamReader { + scanner := bufio.NewScanner(stream) + + initBufferSize := min(4096, maxBufferSize) + + scanner.Buffer(make([]byte, initBufferSize), maxBufferSize) + + split := func(data []byte, atEOF bool) (int, []byte, error) { + if atEOF && len(data) == 0 { + return 0, nil, nil + } + + // We have a full event payload to parse. + if i, nlen := containNewline(data); i >= 0 { + return i + nlen, data[0:i], nil + } + + // If we're at EOF, we have all the data. + if atEOF { + return len(data), data, nil + } + + // Request more data. + + return 0, nil, nil + } + + // Set the split function for the scanning operation. + scanner.Split(split) + + return &StreamReader{ + scanner: scanner, + } +} + +// ReadEvent scans the EventStream for events. +func (r *StreamReader) ReadEvent() ([]byte, error) { + if r.scanner.Scan() { + event := r.scanner.Bytes() + + return event, nil + } + + if err := r.scanner.Err(); err != nil { + if err == context.Canceled { + return nil, io.EOF + } + + return nil, err + } + + return nil, io.EOF +} diff --git a/pkg/providers/cohere/testdata/chat_stream.empty.txt b/pkg/providers/cohere/testdata/chat_stream.empty.txt deleted file mode 100644 index 38471d95..00000000 --- a/pkg/providers/cohere/testdata/chat_stream.empty.txt +++ /dev/null @@ -1,277 +0,0 @@ -{ - "is_finished": false, - "event_type": "stream-start", - "generation_id": "d686011c-e1bb-41c1-9964-823d9b94d394" -} -{ -} -{ - "is_finished": false, - "event_type": "text-generation", - "text": " capital" -} -{ - "is_finished": false, - "event_type": "text-generation", - "text": " of" -} -{ - "is_finished": false, - "event_type": "text-generation", - "text": " the" -} -{ - "is_finished": false, - "event_type": "text-generation", - "text": " United" -} -{ - "is_finished": false, - "event_type": "text-generation", - "text": " Kingdom" -} -{ - "is_finished": false, - "event_type": "text-generation", - "text": " is" -} -{ - "is_finished": false, - "event_type": "text-generation", - "text": " London" -} -{ - "is_finished": false, - "event_type": "text-generation", - "text": "." -} -{ - "is_finished": false, - "event_type": "text-generation", - "text": " London" -} -{ - "is_finished": false, - "event_type": "text-generation", - "text": " is" -} -{ - "is_finished": false, - "event_type": "text-generation", - "text": " a" -} -{ - "is_finished": false, - "event_type": "text-generation", - "text": " vibrant" -} -{ - "is_finished": false, - "event_type": "text-generation", - "text": " city" -} -{ - "is_finished": false, - "event_type": "text-generation", - "text": " full" -} -{ - "is_finished": false, - "event_type": "text-generation", - "text": " of" -} -{ - "is_finished": false, - "event_type": "text-generation", - "text": " diverse" -} -{ - "is_finished": false, - "event_type": "text-generation", - "text": " culture" -} -{ - "is_finished": false, - "event_type": "text-generation", - "text": "," -} -{ - "is_finished": false, - "event_type": "text-generation", - "text": " history" -} -{ - "is_finished": false, - "event_type": "text-generation", - "text": "," -} -{ - "is_finished": false, - "event_type": "text-generation", - "text": " and" -} -{ - "is_finished": false, - "event_type": "text-generation", - "text": " iconic" -} -{ - "is_finished": false, - "event_type": "text-generation", - "text": " landmarks" -} -{ - "is_finished": false, - "event_type": "text-generation", - "text": "." -} -{ - "is_finished": false, - "event_type": "text-generation", - "text": " It" -} -{ - "is_finished": false, - "event_type": "text-generation", - "text": "'s" -} -{ - "is_finished": false, - "event_type": "text-generation", - "text": " a" -} -{ - "is_finished": false, - "event_type": "text-generation", - "text": " global" -} -{ - "is_finished": false, - "event_type": "text-generation", - "text": " city" -} -{ - "is_finished": false, - "event_type": "text-generation", - "text": " that" -} -{ - "is_finished": false, - "event_type": "text-generation", - "text": " has" -} -{ - "is_finished": false, - "event_type": "text-generation", - "text": " influence" -} -{ - "is_finished": false, - "event_type": "text-generation", - "text": " in" -} -{ - "is_finished": false, - "event_type": "text-generation", - "text": " the" -} -{ - "is_finished": false, - "event_type": "text-generation", - "text": " fields" -} -{ - "is_finished": false, - "event_type": "text-generation", - "text": " of" -} -{ - "is_finished": false, - "event_type": "text-generation", - "text": " art" -} -{ - "is_finished": false, - "event_type": "text-generation", - "text": "," -} -{ - "is_finished": false, - "event_type": "text-generation", - "text": " fashion" -} -{ - "is_finished": false, - "event_type": "text-generation", - "text": "," -} -{ - "is_finished": false, - "event_type": "text-generation", - "text": " finance" -} -{ - "is_finished": false, - "event_type": "text-generation", - "text": "," -} -{ - "is_finished": false, - "event_type": "text-generation", - "text": " media" -} -{ - "is_finished": false, - "event_type": "text-generation", - "text": "," -} -{ - "is_finished": false, - "event_type": "text-generation", - "text": " and" -} -{ - "is_finished": false, - "event_type": "text-generation", - "text": " politics" -} -{ - "is_finished": false, - "event_type": "text-generation", - "text": "." -} -{ - "is_finished": true, - "event_type": "stream-end", - "response": { - "response_id": "d4a5e49e-b892-41c5-950d-a97162d19393", - "text": "The capital of the United Kingdom is London. London is a vibrant city full of diverse culture, history, and iconic landmarks. It's a global city that has influence in the fields of art, fashion, finance, media, and politics.", - "generation_id": "d686011c-e1bb-41c1-9964-823d9b94d394", - "chat_history": [ - { - "role": "USER", - "message": "What's the capital of the United Kingdom?" - }, - { - "role": "CHATBOT", - "message": "The capital of the United Kingdom is London. London is a vibrant city full of diverse culture, history, and iconic landmarks. It's a global city that has influence in the fields of art, fashion, finance, media, and politics." - } - ], - "token_count": { - "prompt_tokens": 75, - "response_tokens": 48, - "total_tokens": 123, - "billed_tokens": 57 - }, - "meta": { - "api_version": { - "version": "1" - }, - "billed_units": { - "input_tokens": 9, - "output_tokens": 48 - } - } - }, - "finish_reason": "COMPLETE" -} \ No newline at end of file diff --git a/pkg/providers/cohere/testdata/chat_stream.interrupted.txt b/pkg/providers/cohere/testdata/chat_stream.interrupted.txt new file mode 100644 index 00000000..da0a3abf --- /dev/null +++ b/pkg/providers/cohere/testdata/chat_stream.interrupted.txt @@ -0,0 +1,6 @@ +{"is_finished":false,"event_type":"stream-start","generation_id":"8d246a38-7d24-4dd9-ba78-97be8479db54"} +{"is_finished":false,"event_type":"text-generation","text":"The"} +{"is_finished":false,"event_type":"text-generation","text":" capital"} +{"is_finished":false,"event_type":"text-generation","text":" of"} +{"is_finished":false,"event_type":"text-generation","text":" Greenland"} +{"is_finished":false,"event_type":"text-generation","text":" is"} diff --git a/pkg/providers/cohere/testdata/chat_stream.nodone.txt b/pkg/providers/cohere/testdata/chat_stream.nodone.txt deleted file mode 100644 index 785fe492..00000000 --- a/pkg/providers/cohere/testdata/chat_stream.nodone.txt +++ /dev/null @@ -1,22 +0,0 @@ -data: {"id":"chatcmpl-8wFR3h2Spa9XeRbipfaJczj42pZQg","object":"chat.completion.chunk","created":1708893049,"model":"gpt-3.5-turbo-0125","system_fingerprint":"fp_86156a94a0","choices":[{"index":0,"delta":{"role":"assistant","content":""},"logprobs":null,"finish_reason":null}]} - -data: {"id":"chatcmpl-8wFR3h2Spa9XeRbipfaJczj42pZQg","object":"chat.completion.chunk","created":1708893049,"model":"gpt-3.5-turbo-0125","system_fingerprint":"fp_86156a94a0","choices":[{"index":0,"delta":{"content":"The"},"logprobs":null,"finish_reason":null}]} - -data: {"id":"chatcmpl-8wFR3h2Spa9XeRbipfaJczj42pZQg","object":"chat.completion.chunk","created":1708893049,"model":"gpt-3.5-turbo-0125","system_fingerprint":"fp_86156a94a0","choices":[{"index":0,"delta":{"content":" capital"},"logprobs":null,"finish_reason":null}]} - -data: {"id":"chatcmpl-8wFR3h2Spa9XeRbipfaJczj42pZQg","object":"chat.completion.chunk","created":1708893049,"model":"gpt-3.5-turbo-0125","system_fingerprint":"fp_86156a94a0","choices":[{"index":0,"delta":{"content":" of"},"logprobs":null,"finish_reason":null}]} - -data: {"id":"chatcmpl-8wFR3h2Spa9XeRbipfaJczj42pZQg","object":"chat.completion.chunk","created":1708893049,"model":"gpt-3.5-turbo-0125","system_fingerprint":"fp_86156a94a0","choices":[{"index":0,"delta":{"content":" the"},"logprobs":null,"finish_reason":null}]} - -data: {"id":"chatcmpl-8wFR3h2Spa9XeRbipfaJczj42pZQg","object":"chat.completion.chunk","created":1708893049,"model":"gpt-3.5-turbo-0125","system_fingerprint":"fp_86156a94a0","choices":[{"index":0,"delta":{"content":" United"},"logprobs":null,"finish_reason":null}]} - -data: {"id":"chatcmpl-8wFR3h2Spa9XeRbipfaJczj42pZQg","object":"chat.completion.chunk","created":1708893049,"model":"gpt-3.5-turbo-0125","system_fingerprint":"fp_86156a94a0","choices":[{"index":0,"delta":{"content":" Kingdom"},"logprobs":null,"finish_reason":null}]} - -data: {"id":"chatcmpl-8wFR3h2Spa9XeRbipfaJczj42pZQg","object":"chat.completion.chunk","created":1708893049,"model":"gpt-3.5-turbo-0125","system_fingerprint":"fp_86156a94a0","choices":[{"index":0,"delta":{"content":" is"},"logprobs":null,"finish_reason":null}]} - -data: {"id":"chatcmpl-8wFR3h2Spa9XeRbipfaJczj42pZQg","object":"chat.completion.chunk","created":1708893049,"model":"gpt-3.5-turbo-0125","system_fingerprint":"fp_86156a94a0","choices":[{"index":0,"delta":{"content":" London"},"logprobs":null,"finish_reason":null}]} - -data: {"id":"chatcmpl-8wFR3h2Spa9XeRbipfaJczj42pZQg","object":"chat.completion.chunk","created":1708893049,"model":"gpt-3.5-turbo-0125","system_fingerprint":"fp_86156a94a0","choices":[{"index":0,"delta":{"content":"."},"logprobs":null,"finish_reason":null}]} - -data: {"id":"chatcmpl-8wFR3h2Spa9XeRbipfaJczj42pZQg","object":"chat.completion.chunk","created":1708893049,"model":"gpt-3.5-turbo-0125","system_fingerprint":"fp_86156a94a0","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"stop"}]} - diff --git a/pkg/providers/cohere/testdata/chat_stream.success.txt b/pkg/providers/cohere/testdata/chat_stream.success.txt index d68c7eda..d8a90e12 100644 --- a/pkg/providers/cohere/testdata/chat_stream.success.txt +++ b/pkg/providers/cohere/testdata/chat_stream.success.txt @@ -1,280 +1,31 @@ -{ - "is_finished": false, - "event_type": "stream-start", - "generation_id": "d686011c-e1bb-41c1-9964-823d9b94d394" -} -{ - "is_finished": false, - "event_type": "text-generation", - "text": "The" -} -{ - "is_finished": false, - "event_type": "text-generation", - "text": " capital" -} -{ - "is_finished": false, - "event_type": "text-generation", - "text": " of" -} -{ - "is_finished": false, - "event_type": "text-generation", - "text": " the" -} -{ - "is_finished": false, - "event_type": "text-generation", - "text": " United" -} -{ - "is_finished": false, - "event_type": "text-generation", - "text": " Kingdom" -} -{ - "is_finished": false, - "event_type": "text-generation", - "text": " is" -} -{ - "is_finished": false, - "event_type": "text-generation", - "text": " London" -} -{ - "is_finished": false, - "event_type": "text-generation", - "text": "." -} -{ - "is_finished": false, - "event_type": "text-generation", - "text": " London" -} -{ - "is_finished": false, - "event_type": "text-generation", - "text": " is" -} -{ - "is_finished": false, - "event_type": "text-generation", - "text": " a" -} -{ - "is_finished": false, - "event_type": "text-generation", - "text": " vibrant" -} -{ - "is_finished": false, - "event_type": "text-generation", - "text": " city" -} -{ - "is_finished": false, - "event_type": "text-generation", - "text": " full" -} -{ - "is_finished": false, - "event_type": "text-generation", - "text": " of" -} -{ - "is_finished": false, - "event_type": "text-generation", - "text": " diverse" -} -{ - "is_finished": false, - "event_type": "text-generation", - "text": " culture" -} -{ - "is_finished": false, - "event_type": "text-generation", - "text": "," -} -{ - "is_finished": false, - "event_type": "text-generation", - "text": " history" -} -{ - "is_finished": false, - "event_type": "text-generation", - "text": "," -} -{ - "is_finished": false, - "event_type": "text-generation", - "text": " and" -} -{ - "is_finished": false, - "event_type": "text-generation", - "text": " iconic" -} -{ - "is_finished": false, - "event_type": "text-generation", - "text": " landmarks" -} -{ - "is_finished": false, - "event_type": "text-generation", - "text": "." -} -{ - "is_finished": false, - "event_type": "text-generation", - "text": " It" -} -{ - "is_finished": false, - "event_type": "text-generation", - "text": "'s" -} -{ - "is_finished": false, - "event_type": "text-generation", - "text": " a" -} -{ - "is_finished": false, - "event_type": "text-generation", - "text": " global" -} -{ - "is_finished": false, - "event_type": "text-generation", - "text": " city" -} -{ - "is_finished": false, - "event_type": "text-generation", - "text": " that" -} -{ - "is_finished": false, - "event_type": "text-generation", - "text": " has" -} -{ - "is_finished": false, - "event_type": "text-generation", - "text": " influence" -} -{ - "is_finished": false, - "event_type": "text-generation", - "text": " in" -} -{ - "is_finished": false, - "event_type": "text-generation", - "text": " the" -} -{ - "is_finished": false, - "event_type": "text-generation", - "text": " fields" -} -{ - "is_finished": false, - "event_type": "text-generation", - "text": " of" -} -{ - "is_finished": false, - "event_type": "text-generation", - "text": " art" -} -{ - "is_finished": false, - "event_type": "text-generation", - "text": "," -} -{ - "is_finished": false, - "event_type": "text-generation", - "text": " fashion" -} -{ - "is_finished": false, - "event_type": "text-generation", - "text": "," -} -{ - "is_finished": false, - "event_type": "text-generation", - "text": " finance" -} -{ - "is_finished": false, - "event_type": "text-generation", - "text": "," -} -{ - "is_finished": false, - "event_type": "text-generation", - "text": " media" -} -{ - "is_finished": false, - "event_type": "text-generation", - "text": "," -} -{ - "is_finished": false, - "event_type": "text-generation", - "text": " and" -} -{ - "is_finished": false, - "event_type": "text-generation", - "text": " politics" -} -{ - "is_finished": false, - "event_type": "text-generation", - "text": "." -} -{ - "is_finished": true, - "event_type": "stream-end", - "response": { - "response_id": "d4a5e49e-b892-41c5-950d-a97162d19393", - "text": "The capital of the United Kingdom is London. London is a vibrant city full of diverse culture, history, and iconic landmarks. It's a global city that has influence in the fields of art, fashion, finance, media, and politics.", - "generation_id": "d686011c-e1bb-41c1-9964-823d9b94d394", - "chat_history": [ - { - "role": "USER", - "message": "What's the capital of the United Kingdom?" - }, - { - "role": "CHATBOT", - "message": "The capital of the United Kingdom is London. London is a vibrant city full of diverse culture, history, and iconic landmarks. It's a global city that has influence in the fields of art, fashion, finance, media, and politics." - } - ], - "token_count": { - "prompt_tokens": 75, - "response_tokens": 48, - "total_tokens": 123, - "billed_tokens": 57 - }, - "meta": { - "api_version": { - "version": "1" - }, - "billed_units": { - "input_tokens": 9, - "output_tokens": 48 - } - } - }, - "finish_reason": "COMPLETE" -} \ No newline at end of file +{"is_finished":false,"event_type":"stream-start","generation_id":"8d246a38-7d24-4dd9-ba78-97be8479db54"} +{"is_finished":false,"event_type":"text-generation","text":"The"} +{"is_finished":false,"event_type":"text-generation","text":" capital"} +{"is_finished":false,"event_type":"text-generation","text":" of"} +{"is_finished":false,"event_type":"text-generation","text":" Greenland"} +{"is_finished":false,"event_type":"text-generation","text":" is"} +{"is_finished":false,"event_type":"text-generation","text":" Nu"} +{"is_finished":false,"event_type":"text-generation","text":"uk"} +{"is_finished":false,"event_type":"text-generation","text":"."} +{"is_finished":false,"event_type":"text-generation","text":" It"} +{"is_finished":false,"event_type":"text-generation","text":" is"} +{"is_finished":false,"event_type":"text-generation","text":" the"} +{"is_finished":false,"event_type":"text-generation","text":" biggest"} +{"is_finished":false,"event_type":"text-generation","text":" urban"} +{"is_finished":false,"event_type":"text-generation","text":" area"} +{"is_finished":false,"event_type":"text-generation","text":" and"} +{"is_finished":false,"event_type":"text-generation","text":" serves"} +{"is_finished":false,"event_type":"text-generation","text":" as"} +{"is_finished":false,"event_type":"text-generation","text":" the"} +{"is_finished":false,"event_type":"text-generation","text":" cultural"} +{"is_finished":false,"event_type":"text-generation","text":","} +{"is_finished":false,"event_type":"text-generation","text":" governmental"} +{"is_finished":false,"event_type":"text-generation","text":","} +{"is_finished":false,"event_type":"text-generation","text":" and"} +{"is_finished":false,"event_type":"text-generation","text":" economic"} +{"is_finished":false,"event_type":"text-generation","text":" center"} +{"is_finished":false,"event_type":"text-generation","text":" of"} +{"is_finished":false,"event_type":"text-generation","text":" the"} +{"is_finished":false,"event_type":"text-generation","text":" country"} +{"is_finished":false,"event_type":"text-generation","text":"."} +{"is_finished":true,"event_type":"stream-end","response":{"response_id":"56d2f72e-fd00-4d60-af27-22d62efea204","text":"The capital of Greenland is Nuuk. It is the biggest urban area and serves as the cultural, governmental, and economic center of the country.","generation_id":"8d246a38-7d24-4dd9-ba78-97be8479db54","chat_history":[{"role":"USER","message":"What is the capital of Greenland?"},{"role":"CHATBOT","message":"The capital of Greenland is Nuuk. It is the biggest urban area and serves as the cultural, governmental, and economic center of the country."}],"finish_reason":"COMPLETE","meta":{"api_version":{"version":"1"},"billed_units":{"input_tokens":58,"output_tokens":29},"tokens":{"input_tokens":69,"output_tokens":29}}},"finish_reason":"COMPLETE"} diff --git a/pkg/providers/lang.go b/pkg/providers/lang.go index f4ab258b..7ac96724 100644 --- a/pkg/providers/lang.go +++ b/pkg/providers/lang.go @@ -5,6 +5,8 @@ import ( "io" "time" + "glide/pkg/config/fields" + "glide/pkg/routers/health" "glide/pkg/api/schemas" @@ -40,7 +42,7 @@ type LanguageModel struct { healthTracker *health.Tracker chatLatency *latency.MovingAverage chatStreamLatency *latency.MovingAverage - latencyUpdateInterval *time.Duration + latencyUpdateInterval *fields.Duration } func NewLangModel(modelID string, client LangProvider, budget *health.ErrorBudget, latencyConfig latency.Config, weight int) *LanguageModel { @@ -67,7 +69,7 @@ func (m LanguageModel) Weight() int { return m.weight } -func (m LanguageModel) LatencyUpdateInterval() *time.Duration { +func (m LanguageModel) LatencyUpdateInterval() *fields.Duration { return m.latencyUpdateInterval } @@ -85,19 +87,19 @@ func (m LanguageModel) ChatStreamLatency() *latency.MovingAverage { func (m *LanguageModel) Chat(ctx context.Context, request *schemas.ChatRequest) (*schemas.ChatResponse, error) { startedAt := time.Now() - resp, err := m.client.Chat(ctx, request) - if err == nil { - // record latency per token to normalize measurements - m.chatLatency.Add(float64(time.Since(startedAt)) / float64(resp.ModelResponse.TokenUsage.ResponseTokens)) - - // successful response - resp.ModelID = m.modelID + resp, err := m.client.Chat(ctx, request) + if err != nil { + m.healthTracker.TrackErr(err) return resp, err } - m.healthTracker.TrackErr(err) + // record latency per token to normalize measurements + m.chatLatency.Add(float64(time.Since(startedAt)) / float64(resp.ModelResponse.TokenUsage.ResponseTokens)) + + // successful response + resp.ModelID = m.modelID return resp, err } @@ -149,6 +151,8 @@ func (m *LanguageModel) ChatStream(ctx context.Context, req *schemas.ChatStreamR return } + chunk.ModelID = m.modelID + streamResultC <- clients.NewChatStreamResult(chunk, nil) if chunkLatency > 1*time.Millisecond { diff --git a/pkg/providers/openai/chat.go b/pkg/providers/openai/chat.go index fcd75db9..b292c664 100644 --- a/pkg/providers/openai/chat.go +++ b/pkg/providers/openai/chat.go @@ -84,9 +84,8 @@ func (c *Client) doChatRequest(ctx context.Context, payload *ChatRequest) (*sche req.Header.Set("Authorization", fmt.Sprintf("Bearer %v", string(c.config.APIKey))) // TODO: this could leak information from messages which may not be a desired thing to have - c.tel.Logger.Debug( + c.logger.Debug( "Chat Request", - zap.String("provider", c.Provider()), zap.String("chatURL", c.chatURL), zap.Any("payload", payload), ) @@ -105,9 +104,9 @@ func (c *Client) doChatRequest(ctx context.Context, payload *ChatRequest) (*sche // Read the response body into a byte slice bodyBytes, err := io.ReadAll(resp.Body) if err != nil { - c.tel.Logger.Error( + c.logger.Error( "Failed to read chat response", - zap.String("provider", c.Provider()), zap.Error(err), + zap.Error(err), zap.ByteString("rawResponse", bodyBytes), ) @@ -119,9 +118,8 @@ func (c *Client) doChatRequest(ctx context.Context, payload *ChatRequest) (*sche err = json.Unmarshal(bodyBytes, &chatCompletion) if err != nil { - c.tel.Logger.Error( + c.logger.Error( "Failed to unmarshal chat response", - zap.String("provider", c.Provider()), zap.ByteString("rawResponse", bodyBytes), zap.Error(err), ) diff --git a/pkg/providers/openai/chat_stream.go b/pkg/providers/openai/chat_stream.go index fb8be776..0d33f52e 100644 --- a/pkg/providers/openai/chat_stream.go +++ b/pkg/providers/openai/chat_stream.go @@ -10,45 +10,37 @@ import ( "github.com/r3labs/sse/v2" "glide/pkg/providers/clients" - "glide/pkg/telemetry" - "go.uber.org/zap" "glide/pkg/api/schemas" ) -var ( - StopReason = "stop" - streamDoneMarker = []byte("[DONE]") -) +var StreamDoneMarker = []byte("[DONE]") // ChatStream represents OpenAI chat stream for a specific request type ChatStream struct { - tel *telemetry.Telemetry - client *http.Client - req *http.Request - reqID string - reqMetadata *schemas.Metadata - resp *http.Response - reader *sse.EventStreamReader - errMapper *ErrorMapper + client *http.Client + req *http.Request + resp *http.Response + reader *sse.EventStreamReader + finishReasonMapper *FinishReasonMapper + errMapper *ErrorMapper + logger *zap.Logger } func NewChatStream( - tel *telemetry.Telemetry, client *http.Client, req *http.Request, - reqID string, - reqMetadata *schemas.Metadata, + finishReasonMapper *FinishReasonMapper, errMapper *ErrorMapper, + logger *zap.Logger, ) *ChatStream { return &ChatStream{ - tel: tel, - client: client, - req: req, - reqID: reqID, - reqMetadata: reqMetadata, - errMapper: errMapper, + client: client, + req: req, + finishReasonMapper: finishReasonMapper, + errMapper: errMapper, + logger: logger, } } @@ -74,9 +66,8 @@ func (s *ChatStream) Recv() (*schemas.ChatStreamChunk, error) { for { rawEvent, err := s.reader.ReadEvent() if err != nil { - s.tel.L().Warn( + s.logger.Warn( "Chat stream is unexpectedly disconnected", - zap.String("provider", providerName), zap.Error(err), ) @@ -86,15 +77,14 @@ func (s *ChatStream) Recv() (*schemas.ChatStreamChunk, error) { return nil, clients.ErrProviderUnavailable } - s.tel.L().Debug( + s.logger.Debug( "Raw chat stream chunk", - zap.String("provider", providerName), zap.ByteString("rawChunk", rawEvent), ) event, err := clients.ParseSSEvent(rawEvent) - if bytes.Equal(event.Data, streamDoneMarker) { + if bytes.Equal(event.Data, StreamDoneMarker) { return nil, io.EOF } @@ -103,9 +93,8 @@ func (s *ChatStream) Recv() (*schemas.ChatStreamChunk, error) { } if !event.HasContent() { - s.tel.L().Debug( + s.logger.Debug( "Received an empty message in chat stream, skipping it", - zap.String("provider", providerName), zap.Any("msg", event), ) @@ -119,30 +108,23 @@ func (s *ChatStream) Recv() (*schemas.ChatStreamChunk, error) { responseChunk := completionChunk.Choices[0] - var finishReason *schemas.FinishReason - - if responseChunk.FinishReason == StopReason { - finishReason = &schemas.Complete - } - // TODO: use objectpool here return &schemas.ChatStreamChunk{ - ID: s.reqID, - Provider: providerName, Cached: false, + Provider: providerName, ModelName: completionChunk.ModelName, - Metadata: s.reqMetadata, ModelResponse: schemas.ModelChunkResponse{ Metadata: &schemas.Metadata{ "response_id": completionChunk.ID, "system_fingerprint": completionChunk.SystemFingerprint, + "generated_at": completionChunk.Created, }, Message: schemas.ChatMessage{ Role: responseChunk.Delta.Role, Content: responseChunk.Delta.Content, }, - FinishReason: finishReason, }, + FinishReason: s.finishReasonMapper.Map(responseChunk.FinishReason), }, nil } } @@ -167,12 +149,11 @@ func (c *Client) ChatStream(ctx context.Context, req *schemas.ChatStreamRequest) } return NewChatStream( - c.tel, c.httpClient, httpRequest, - req.ID, - req.Metadata, + c.finishReasonMapper, c.errMapper, + c.logger, ), nil } @@ -214,7 +195,7 @@ func (c *Client) makeStreamReq(ctx context.Context, req *schemas.ChatStreamReque request.Header.Set("Connection", "keep-alive") // TODO: this could leak information from messages which may not be a desired thing to have - c.tel.L().Debug( + c.logger.Debug( "Stream chat request", zap.String("chatURL", c.chatURL), zap.Any("payload", chatRequest), diff --git a/pkg/providers/openai/client.go b/pkg/providers/openai/client.go index 22d68d45..39af0d63 100644 --- a/pkg/providers/openai/client.go +++ b/pkg/providers/openai/client.go @@ -5,6 +5,8 @@ import ( "net/http" "net/url" + "go.uber.org/zap" + "glide/pkg/providers/clients" "glide/pkg/telemetry" ) @@ -24,9 +26,11 @@ type Client struct { chatURL string chatRequestTemplate *ChatRequest errMapper *ErrorMapper + finishReasonMapper *FinishReasonMapper config *Config httpClient *http.Client tel *telemetry.Telemetry + logger *zap.Logger } // NewClient creates a new OpenAI client for the OpenAI API. @@ -36,11 +40,16 @@ func NewClient(providerConfig *Config, clientConfig *clients.ClientConfig, tel * return nil, err } + logger := tel.L().With( + zap.String("provider", providerName), + ) + c := &Client{ baseURL: providerConfig.BaseURL, chatURL: chatURL, config: providerConfig, chatRequestTemplate: NewChatRequestFromConfig(providerConfig), + finishReasonMapper: NewFinishReasonMapper(tel), errMapper: NewErrorMapper(tel), httpClient: &http.Client{ Timeout: *clientConfig.Timeout, @@ -50,7 +59,8 @@ func NewClient(providerConfig *Config, clientConfig *clients.ClientConfig, tel * MaxIdleConnsPerHost: 2, }, }, - tel: tel, + tel: tel, + logger: logger, } return c, nil diff --git a/pkg/providers/openai/finish_reasons.go b/pkg/providers/openai/finish_reasons.go new file mode 100644 index 00000000..502d8725 --- /dev/null +++ b/pkg/providers/openai/finish_reasons.go @@ -0,0 +1,51 @@ +package openai + +import ( + "glide/pkg/api/schemas" + "glide/pkg/telemetry" + "go.uber.org/zap" +) + +var ( + // Reference: https://platform.openai.com/docs/api-reference/chat/object + + CompleteReason = "stop" + MaxTokensReason = "length" + FilteredReason = "content_filter" +) + +func NewFinishReasonMapper(tel *telemetry.Telemetry) *FinishReasonMapper { + return &FinishReasonMapper{ + tel: tel, + } +} + +type FinishReasonMapper struct { + tel *telemetry.Telemetry +} + +func (m *FinishReasonMapper) Map(finishReason string) *schemas.FinishReason { + if len(finishReason) == 0 { + return nil + } + + var reason *schemas.FinishReason + + switch finishReason { + case CompleteReason: + reason = &schemas.Complete + case MaxTokensReason: + reason = &schemas.MaxTokens + case FilteredReason: + reason = &schemas.ContentFiltered + default: + m.tel.Logger.Warn( + "Unknown finish reason, other is going to used", + zap.String("unknown_reason", finishReason), + ) + + reason = &schemas.OtherReason + } + + return reason +} diff --git a/pkg/providers/provider.go b/pkg/providers/provider.go index 0fffa08b..9d7bf9c7 100644 --- a/pkg/providers/provider.go +++ b/pkg/providers/provider.go @@ -1,7 +1,7 @@ package providers import ( - "time" + "glide/pkg/config/fields" ) // ModelProvider exposes provider context @@ -13,6 +13,6 @@ type ModelProvider interface { type Model interface { ID() string Healthy() bool - LatencyUpdateInterval() *time.Duration + LatencyUpdateInterval() *fields.Duration Weight() int } diff --git a/pkg/providers/testing/lang.go b/pkg/providers/testing/lang.go index 405b3c53..00389dac 100644 --- a/pkg/providers/testing/lang.go +++ b/pkg/providers/testing/lang.go @@ -30,7 +30,6 @@ func (m *RespMock) Resp() *schemas.ChatResponse { func (m *RespMock) RespChunk() *schemas.ChatStreamChunk { return &schemas.ChatStreamChunk{ - ID: "rsp0001", ModelResponse: schemas.ModelChunkResponse{ Message: schemas.ChatMessage{ Content: m.Msg, diff --git a/pkg/providers/testing/models.go b/pkg/providers/testing/models.go index c5f8d6d6..712fcaf0 100644 --- a/pkg/providers/testing/models.go +++ b/pkg/providers/testing/models.go @@ -3,6 +3,8 @@ package testing import ( "time" + "glide/pkg/config/fields" + "glide/pkg/providers" "glide/pkg/routers/latency" ) @@ -42,10 +44,10 @@ func (m *LangModelMock) ChatLatency() *latency.MovingAverage { return m.chatLatency } -func (m LangModelMock) LatencyUpdateInterval() *time.Duration { +func (m LangModelMock) LatencyUpdateInterval() *fields.Duration { updateInterval := 30 * time.Second - return &updateInterval + return (*fields.Duration)(&updateInterval) } func (m LangModelMock) Weight() int { diff --git a/pkg/routers/config.go b/pkg/routers/config.go index f17ed52e..19633688 100644 --- a/pkg/routers/config.go +++ b/pkg/routers/config.go @@ -12,7 +12,7 @@ import ( ) type Config struct { - LanguageRouters []LangRouterConfig `yaml:"language" validate:"required,min=1"` // the list of language routers + LanguageRouters []LangRouterConfig `yaml:"language" validate:"required,gte=1,dive"` // the list of language routers } func (c *Config) BuildLangRouters(tel *telemetry.Telemetry) ([]*LangRouter, error) { @@ -59,7 +59,7 @@ type LangRouterConfig struct { Enabled bool `yaml:"enabled" json:"enabled" validate:"required"` // Is router enabled? Retry *retry.ExpRetryConfig `yaml:"retry" json:"retry" validate:"required"` // retry when no healthy model is available to router RoutingStrategy routing.Strategy `yaml:"strategy" json:"strategy" swaggertype:"primitive,string" validate:"required"` // strategy on picking the next model to serve the request - Models []providers.LangModelConfig `yaml:"models" json:"models" validate:"required,min=1"` // the list of models that could handle requests + Models []providers.LangModelConfig `yaml:"models" json:"models" validate:"required,min=1,dive"` // the list of models that could handle requests } // BuildModels creates LanguageModel slice out of the given config @@ -106,7 +106,7 @@ func (c *LangRouterConfig) BuildModels(tel *telemetry.Telemetry) ([]*providers.L chatModels = append(chatModels, model) if !model.SupportChatStream() { - tel.L().Warn( + tel.L().WithOptions(zap.AddStacktrace(zap.ErrorLevel)).Warn( "Provider doesn't support or have not been yet integrated with streaming chat, it won't serve streaming chat requests", zap.String("routerID", c.ID), zap.String("modelID", model.ID()), diff --git a/pkg/routers/latency/config.go b/pkg/routers/latency/config.go index dd1001f8..c78cc7aa 100644 --- a/pkg/routers/latency/config.go +++ b/pkg/routers/latency/config.go @@ -1,12 +1,16 @@ package latency -import "time" +import ( + "time" + + "glide/pkg/config/fields" +) // Config defines setting for moving average latency calculations type Config struct { - Decay float64 `yaml:"decay" json:"decay"` // Weight of new latency measurements - WarmupSamples uint8 `yaml:"warmup_samples" json:"warmup_samples"` // The number of latency probes required to init moving average - UpdateInterval *time.Duration `yaml:"update_interval,omitempty" json:"update_interval" swaggertype:"primitive,string"` // How often gateway should probe models with not the lowest response latency + Decay float64 `yaml:"decay" json:"decay"` // Weight of new latency measurements + WarmupSamples uint8 `yaml:"warmup_samples" json:"warmup_samples"` // The number of latency probes required to init moving average + UpdateInterval *fields.Duration `yaml:"update_interval,omitempty" json:"update_interval" swaggertype:"primitive,string"` // How often gateway should probe models with not the lowest response latency } func DefaultConfig() *Config { @@ -15,6 +19,6 @@ func DefaultConfig() *Config { return &Config{ Decay: 0.06, WarmupSamples: 3, - UpdateInterval: &defaultUpdateInterval, + UpdateInterval: (*fields.Duration)(&defaultUpdateInterval), } } diff --git a/pkg/routers/router.go b/pkg/routers/router.go index aa8ba067..fa9b9ef0 100644 --- a/pkg/routers/router.go +++ b/pkg/routers/router.go @@ -19,8 +19,10 @@ var ( ErrNoModelAvailable = errors.New("could not handle request because all providers are not available") ) +type RouterID = string + type LangRouter struct { - routerID string + routerID RouterID Config *LangRouterConfig chatModels []*providers.LanguageModel chatStreamModels []*providers.LanguageModel @@ -28,6 +30,7 @@ type LangRouter struct { chatStreamRouting routing.LangModelRouting retry *retry.ExpRetry tel *telemetry.Telemetry + logger *zap.Logger } func NewLangRouter(cfg *LangRouterConfig, tel *telemetry.Telemetry) (*LangRouter, error) { @@ -50,12 +53,13 @@ func NewLangRouter(cfg *LangRouterConfig, tel *telemetry.Telemetry) (*LangRouter chatRouting: chatRouting, chatStreamRouting: chatStreamRouting, tel: tel, + logger: tel.L().With(zap.String("routerID", cfg.ID)), } return router, err } -func (r *LangRouter) ID() string { +func (r *LangRouter) ID() RouterID { return r.routerID } @@ -89,9 +93,8 @@ func (r *LangRouter) Chat(ctx context.Context, req *schemas.ChatRequest) (*schem resp, err := langModel.Chat(ctx, req) if err != nil { - r.tel.L().Warn( + r.logger.Warn( "Lang model failed processing chat request", - zap.String("routerID", r.ID()), zap.String("modelID", langModel.ID()), zap.String("provider", langModel.Provider()), zap.Error(err), @@ -107,7 +110,7 @@ func (r *LangRouter) Chat(ctx context.Context, req *schemas.ChatRequest) (*schem // no providers were available to handle the request, // so we have to wait a bit with a hope there is some available next time - r.tel.L().Warn("No healthy model found to serve chat request, wait and retry", zap.String("routerID", r.ID())) + r.logger.Warn("No healthy model found to serve chat request, wait and retry") err := retryIterator.WaitNext(ctx) if err != nil { @@ -117,7 +120,7 @@ func (r *LangRouter) Chat(ctx context.Context, req *schemas.ChatRequest) (*schem } // if we reach this part, then we are in trouble - r.tel.L().Error("No model was available to handle chat request", zap.String("routerID", r.ID())) + r.logger.Error("No model was available to handle chat request") return nil, ErrNoModelAvailable } @@ -125,15 +128,17 @@ func (r *LangRouter) Chat(ctx context.Context, req *schemas.ChatRequest) (*schem func (r *LangRouter) ChatStream( ctx context.Context, req *schemas.ChatStreamRequest, - respC chan<- *schemas.ChatStreamResult, + respC chan<- *schemas.ChatStreamMessage, ) { if len(r.chatStreamModels) == 0 { - respC <- schemas.NewChatStreamErrorResult(&schemas.ChatStreamError{ - ID: req.ID, - ErrCode: "noModels", - Message: ErrNoModels.Error(), - Metadata: req.Metadata, - }) + respC <- schemas.NewChatStreamError( + req.ID, + r.routerID, + schemas.NoModelConfigured, + ErrNoModels.Error(), + req.Metadata, + &schemas.ErrorReason, + ) return } @@ -155,9 +160,8 @@ func (r *LangRouter) ChatStream( langModel := model.(providers.LangModel) modelRespC, err := langModel.ChatStream(ctx, req) if err != nil { - r.tel.L().Error( + r.logger.Error( "Lang model failed to create streaming chat request", - zap.String("routerID", r.ID()), zap.String("modelID", langModel.ID()), zap.String("provider", langModel.Provider()), zap.Error(err), @@ -169,9 +173,8 @@ func (r *LangRouter) ChatStream( for chunkResult := range modelRespC { err = chunkResult.Error() if err != nil { - r.tel.L().Warn( + r.logger.Warn( "Lang model failed processing streaming chat request", - zap.String("routerID", r.ID()), zap.String("modelID", langModel.ID()), zap.String("provider", langModel.Provider()), zap.Error(err), @@ -180,17 +183,26 @@ func (r *LangRouter) ChatStream( // It's challenging to hide an error in case of streaming chat as consumer apps // may have already used all chunks we streamed this far (e.g. showed them to their users like OpenAI UI does), // so we cannot easily restart that process from scratch - respC <- schemas.NewChatStreamErrorResult(&schemas.ChatStreamError{ - ID: req.ID, - ErrCode: "modelUnavailable", - Message: err.Error(), - Metadata: req.Metadata, - }) + respC <- schemas.NewChatStreamError( + req.ID, + r.routerID, + schemas.ModelUnavailable, + err.Error(), + req.Metadata, + nil, + ) continue NextModel } - respC <- schemas.NewChatStreamResult(chunkResult.Chunk()) + chunk := chunkResult.Chunk() + + respC <- schemas.NewChatStreamChunk( + req.ID, + r.routerID, + req.Metadata, + chunk, + ) } return @@ -198,35 +210,36 @@ func (r *LangRouter) ChatStream( // no providers were available to handle the request, // so we have to wait a bit with a hope there is some available next time - r.tel.L().Warn( - "No healthy model found to serve streaming chat request, wait and retry", - zap.String("routerID", r.ID()), - ) + r.logger.Warn("No healthy model found to serve streaming chat request, wait and retry") err := retryIterator.WaitNext(ctx) if err != nil { // something has cancelled the context - respC <- schemas.NewChatStreamErrorResult(&schemas.ChatStreamError{ - ID: req.ID, - ErrCode: "other", - Message: err.Error(), - Metadata: req.Metadata, - }) + respC <- schemas.NewChatStreamError( + req.ID, + r.routerID, + schemas.UnknownError, + err.Error(), + req.Metadata, + nil, + ) return } } // if we reach this part, then we are in trouble - r.tel.L().Error( - "No model was available to handle streaming chat request. Try to configure more fallback models to avoid this", - zap.String("routerID", r.ID()), + r.logger.Error( + "No model was available to handle streaming chat request. " + + "Try to configure more fallback models to avoid this", ) - respC <- schemas.NewChatStreamErrorResult(&schemas.ChatStreamError{ - ID: req.ID, - ErrCode: "allModelsUnavailable", - Message: ErrNoModelAvailable.Error(), - Metadata: req.Metadata, - }) + respC <- schemas.NewChatStreamError( + req.ID, + r.routerID, + schemas.AllModelsUnavailable, + ErrNoModelAvailable.Error(), + req.Metadata, + &schemas.ErrorReason, + ) } diff --git a/pkg/routers/router_test.go b/pkg/routers/router_test.go index 4fd50d28..d5fa8640 100644 --- a/pkg/routers/router_test.go +++ b/pkg/routers/router_test.go @@ -110,6 +110,7 @@ func TestLangRouter_Chat_PickThirdHealthy(t *testing.T) { chatModels: langModels, chatStreamModels: langModels, tel: telemetry.NewTelemetryMock(), + logger: telemetry.NewLoggerMock(), } ctx := context.Background() @@ -158,6 +159,7 @@ func TestLangRouter_Chat_SuccessOnRetry(t *testing.T) { chatModels: langModels, chatStreamModels: langModels, tel: telemetry.NewTelemetryMock(), + logger: telemetry.NewLoggerMock(), } resp, err := router.Chat(context.Background(), schemas.NewChatFromStr("tell me a dad joke")) @@ -201,6 +203,7 @@ func TestLangRouter_Chat_UnhealthyModelInThePool(t *testing.T) { chatStreamModels: langModels, chatStreamRouting: routing.NewPriority(models), tel: telemetry.NewTelemetryMock(), + logger: telemetry.NewLoggerMock(), } for i := 0; i < 2; i++ { @@ -246,6 +249,7 @@ func TestLangRouter_Chat_AllModelsUnavailable(t *testing.T) { chatStreamModels: langModels, chatStreamRouting: routing.NewPriority(models), tel: telemetry.NewTelemetryMock(), + logger: telemetry.NewLoggerMock(), } _, err := router.Chat(context.Background(), schemas.NewChatFromStr("tell me a dad joke")) @@ -302,11 +306,12 @@ func TestLangRouter_ChatStream(t *testing.T) { chatStreamRouting: routing.NewPriority(models), chatStreamModels: langModels, tel: telemetry.NewTelemetryMock(), + logger: telemetry.NewLoggerMock(), } ctx := context.Background() req := schemas.NewChatStreamFromStr("tell me a dad joke") - respC := make(chan *schemas.ChatStreamResult) + respC := make(chan *schemas.ChatStreamMessage) defer close(respC) @@ -316,11 +321,12 @@ func TestLangRouter_ChatStream(t *testing.T) { for range 5 { select { //nolint:gosimple - case chunk := <-respC: - require.Nil(t, chunk.Error()) - require.NotNil(t, chunk.Chunk().ModelResponse.Message.Content) + case message := <-respC: + require.Nil(t, message.Error) + require.NotNil(t, message.Chunk) + require.NotNil(t, message.Chunk.ModelResponse.Message.Content) - chunks = append(chunks, chunk.Chunk().ModelResponse.Message.Content) + chunks = append(chunks, message.Chunk.ModelResponse.Message.Content) } } @@ -370,11 +376,12 @@ func TestLangRouter_ChatStream_FailOnFirst(t *testing.T) { chatStreamRouting: routing.NewPriority(models), chatStreamModels: langModels, tel: telemetry.NewTelemetryMock(), + logger: telemetry.NewLoggerMock(), } ctx := context.Background() req := schemas.NewChatStreamFromStr("tell me a dad joke") - respC := make(chan *schemas.ChatStreamResult) + respC := make(chan *schemas.ChatStreamMessage) defer close(respC) @@ -384,11 +391,12 @@ func TestLangRouter_ChatStream_FailOnFirst(t *testing.T) { for range 3 { select { //nolint:gosimple - case chunk := <-respC: - require.Nil(t, chunk.Error()) - require.NotNil(t, chunk.Chunk().ModelResponse.Message.Content) + case message := <-respC: + require.Nil(t, message.Error) + require.NotNil(t, message.Chunk.ModelResponse.Message.Content) + require.NotNil(t, message.Chunk.ModelResponse.Message.Content) - chunks = append(chunks, chunk.Chunk().ModelResponse.Message.Content) + chunks = append(chunks, message.Chunk.ModelResponse.Message.Content) } } @@ -438,9 +446,10 @@ func TestLangRouter_ChatStream_AllModelsUnavailable(t *testing.T) { chatStreamModels: langModels, chatStreamRouting: routing.NewPriority(models), tel: telemetry.NewTelemetryMock(), + logger: telemetry.NewLoggerMock(), } - respC := make(chan *schemas.ChatStreamResult) + respC := make(chan *schemas.ChatStreamMessage) defer close(respC) go router.ChatStream(context.Background(), schemas.NewChatStreamFromStr("tell me a dad joke"), respC) @@ -449,10 +458,11 @@ func TestLangRouter_ChatStream_AllModelsUnavailable(t *testing.T) { for range 3 { result := <-respC - require.Nil(t, result.Chunk()) + require.Nil(t, result.Chunk) + require.NotNil(t, result.Error) - errs = append(errs, result.Error().ErrCode) + errs = append(errs, result.Error.ErrCode) } - require.Equal(t, []string{"modelUnavailable", "modelUnavailable", "allModelsUnavailable"}, errs) + require.Equal(t, []string{schemas.ModelUnavailable, schemas.ModelUnavailable, schemas.AllModelsUnavailable}, errs) } diff --git a/pkg/routers/routing/least_latency.go b/pkg/routers/routing/least_latency.go index 422fd863..78e04748 100644 --- a/pkg/routers/routing/least_latency.go +++ b/pkg/routers/routing/least_latency.go @@ -53,7 +53,7 @@ func (s *ModelSchedule) Update() { s.mu.Lock() defer s.mu.Unlock() - s.expireAt = time.Now().Add(*s.model.LatencyUpdateInterval()) + s.expireAt = time.Now().Add(time.Duration(*s.model.LatencyUpdateInterval())) } // LeastLatencyRouting routes requests to the model that responses the fastest diff --git a/pkg/telemetry/telemetry.go b/pkg/telemetry/telemetry.go index a97f87ac..ec85fec4 100644 --- a/pkg/telemetry/telemetry.go +++ b/pkg/telemetry/telemetry.go @@ -35,10 +35,14 @@ func NewTelemetry(cfg *Config) (*Telemetry, error) { }, nil } +func NewLoggerMock() *zap.Logger { + return zap.NewNop() +} + // NewTelemetryMock returns Telemetry object with NoOp loggers, meters, tracers func NewTelemetryMock() *Telemetry { return &Telemetry{ Config: DefaultConfig(), - Logger: zap.NewNop(), + Logger: NewLoggerMock(), } }