Skip to content

Commit

Permalink
#43 Updated Swagger specs
Browse files Browse the repository at this point in the history
  • Loading branch information
@roma-glushko
roma-glushko committed Jan 8, 2024
1 parent 3cd26b5 commit 9008a33
Show file tree
Hide file tree
Showing 7 changed files with 160 additions and 46 deletions.
65 changes: 54 additions & 11 deletions docs/docs.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,14 @@ const docTemplate = `{
}
},
"definitions": {
"clients.ClientConfig": {
"type": "object",
"properties": {
"timeout": {
"type": "integer"
}
}
},
"http.ErrorSchema": {
"type": "object",
"properties": {
Expand Down Expand Up @@ -231,16 +239,35 @@ const docTemplate = `{
"id"
],
"properties": {
"client": {
"$ref": "#/definitions/clients.ClientConfig"
},
"enabled": {
"description": "Is the model enabled?",
"type": "boolean"
},
"error_budget": {
"type": "string"
},
"id": {
"description": "Model instance ID (unique in scope of the router)",
"type": "string"
},
"openai": {
"$ref": "#/definitions/openai.Config"
}
}
},
"retry.ExpRetryConfig": {
"type": "object",
"properties": {
"max_delay": {
"type": "integer"
},
"timeout": {
"max_retries": {
"type": "integer"
},
"min_delay": {
"type": "integer"
}
}
Expand All @@ -253,22 +280,47 @@ const docTemplate = `{
],
"properties": {
"enabled": {
"description": "Is router enabled?",
"type": "boolean"
},
"models": {
"description": "the list of models that could handle requests",
"type": "array",
"items": {
"$ref": "#/definitions/providers.LangModelConfig"
}
},
"retry": {
"description": "TODO: how to specify other backoff strategies?",
"allOf": [
{
"$ref": "#/definitions/retry.ExpRetryConfig"
}
]
},
"routers": {
"description": "Unique router ID",
"type": "string"
},
"strategy": {
"$ref": "#/definitions/strategy.RoutingStrategy"
"description": "strategy on picking the next model to serve the request",
"allOf": [
{
"$ref": "#/definitions/routing.Strategy"
}
]
}
}
},
"routing.Strategy": {
"type": "string",
"enum": [
"priority"
],
"x-enum-varnames": [
"Priority"
]
},
"schemas.ChatMessage": {
"type": "object",
"properties": {
Expand Down Expand Up @@ -356,15 +408,6 @@ const docTemplate = `{
"type": "string"
}
}
},
"strategy.RoutingStrategy": {
"type": "string",
"enum": [
"priority"
],
"x-enum-varnames": [
"Priority"
]
}
}
}`
Expand Down
65 changes: 54 additions & 11 deletions docs/swagger.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -122,6 +122,14 @@
}
},
"definitions": {
"clients.ClientConfig": {
"type": "object",
"properties": {
"timeout": {
"type": "integer"
}
}
},
"http.ErrorSchema": {
"type": "object",
"properties": {
Expand Down Expand Up @@ -228,16 +236,35 @@
"id"
],
"properties": {
"client": {
"$ref": "#/definitions/clients.ClientConfig"
},
"enabled": {
"description": "Is the model enabled?",
"type": "boolean"
},
"error_budget": {
"type": "string"
},
"id": {
"description": "Model instance ID (unique in scope of the router)",
"type": "string"
},
"openai": {
"$ref": "#/definitions/openai.Config"
}
}
},
"retry.ExpRetryConfig": {
"type": "object",
"properties": {
"max_delay": {
"type": "integer"
},
"timeout": {
"max_retries": {
"type": "integer"
},
"min_delay": {
"type": "integer"
}
}
Expand All @@ -250,22 +277,47 @@
],
"properties": {
"enabled": {
"description": "Is router enabled?",
"type": "boolean"
},
"models": {
"description": "the list of models that could handle requests",
"type": "array",
"items": {
"$ref": "#/definitions/providers.LangModelConfig"
}
},
"retry": {
"description": "TODO: how to specify other backoff strategies?",
"allOf": [
{
"$ref": "#/definitions/retry.ExpRetryConfig"
}
]
},
"routers": {
"description": "Unique router ID",
"type": "string"
},
"strategy": {
"$ref": "#/definitions/strategy.RoutingStrategy"
"description": "strategy on picking the next model to serve the request",
"allOf": [
{
"$ref": "#/definitions/routing.Strategy"
}
]
}
}
},
"routing.Strategy": {
"type": "string",
"enum": [
"priority"
],
"x-enum-varnames": [
"Priority"
]
},
"schemas.ChatMessage": {
"type": "object",
"properties": {
Expand Down Expand Up @@ -353,15 +405,6 @@
"type": "string"
}
}
},
"strategy.RoutingStrategy": {
"type": "string",
"enum": [
"priority"
],
"x-enum-varnames": [
"Priority"
]
}
}
}
45 changes: 36 additions & 9 deletions docs/swagger.yaml
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,10 @@
basePath: /
definitions:
clients.ClientConfig:
properties:
timeout:
type: integer
type: object
http.ErrorSchema:
properties:
message:
Expand Down Expand Up @@ -68,33 +73,61 @@ definitions:
type: object
providers.LangModelConfig:
properties:
client:
$ref: '#/definitions/clients.ClientConfig'
enabled:
description: Is the model enabled?
type: boolean
error_budget:
type: string
id:
description: Model instance ID (unique in scope of the router)
type: string
openai:
$ref: '#/definitions/openai.Config'
timeout:
type: integer
required:
- id
type: object
retry.ExpRetryConfig:
properties:
max_delay:
type: integer
max_retries:
type: integer
min_delay:
type: integer
type: object
routers.LangRouterConfig:
properties:
enabled:
description: Is router enabled?
type: boolean
models:
description: the list of models that could handle requests
items:
$ref: '#/definitions/providers.LangModelConfig'
type: array
retry:
allOf:
- $ref: '#/definitions/retry.ExpRetryConfig'
description: 'TODO: how to specify other backoff strategies?'
routers:
description: Unique router ID
type: string
strategy:
$ref: '#/definitions/strategy.RoutingStrategy'
allOf:
- $ref: '#/definitions/routing.Strategy'
description: strategy on picking the next model to serve the request
required:
- models
- routers
type: object
routing.Strategy:
enum:
- priority
type: string
x-enum-varnames:
- Priority
schemas.ChatMessage:
properties:
content:
Expand Down Expand Up @@ -156,12 +189,6 @@ definitions:
router:
type: string
type: object
strategy.RoutingStrategy:
enum:
- priority
type: string
x-enum-varnames:
- Priority
host: localhost:9099
info:
contact:
Expand Down
6 changes: 3 additions & 3 deletions pkg/providers/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,9 @@ import (
var ErrProviderNotFound = errors.New("provider not found")

type LangModelConfig struct {
ID string `yaml:"id" json:"id" validate:"required"`
Enabled bool `yaml:"enabled" json:"enabled"`
ErrorBudget health.ErrorBudget `yaml:"error_budget" json:"error_budget"`
ID string `yaml:"id" json:"id" validate:"required"` // Model instance ID (unique in scope of the router)
Enabled bool `yaml:"enabled" json:"enabled"` // Is the model enabled?
ErrorBudget health.ErrorBudget `yaml:"error_budget" json:"error_budget" swaggertype:"primitive,string"`
Client *clients.ClientConfig `yaml:"client" json:"client"`
OpenAI *openai.Config `yaml:"openai" json:"openai"`
// Add other providers like
Expand Down
13 changes: 7 additions & 6 deletions pkg/routers/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ import (
)

type Config struct {
LanguageRouters []LangRouterConfig `yaml:"language"`
LanguageRouters []LangRouterConfig `yaml:"language"` // the list of language routers
}

func (c *Config) BuildLangRouters(tel *telemetry.Telemetry) ([]*LangRouter, error) {
Expand Down Expand Up @@ -43,11 +43,12 @@ func (c *Config) BuildLangRouters(tel *telemetry.Telemetry) ([]*LangRouter, erro
}

type LangRouterConfig struct {
ID string `yaml:"id" json:"routers" validate:"required"`
Enabled bool `yaml:"enabled" json:"enabled"`
Retry *retry.ExpRetryConfig `yaml:"retry" json:"retry"` // TODO: how to specify other backoff strategies?
RoutingStrategy routing.Strategy `yaml:"strategy" json:"strategy"`
Models []providers.LangModelConfig `yaml:"models" json:"models" validate:"required"`
ID string `yaml:"id" json:"routers" validate:"required"` // Unique router ID
Enabled bool `yaml:"enabled" json:"enabled"` // Is router enabled?
// TODO: how to specify other backoff strategies?
Retry *retry.ExpRetryConfig `yaml:"retry" json:"retry"` // retry when no healthy model is available to router
RoutingStrategy routing.Strategy `yaml:"strategy" json:"strategy"` // strategy on picking the next model to serve the request
Models []providers.LangModelConfig `yaml:"models" json:"models" validate:"required"` // the list of models that could handle requests
}

// BuildModels creates LanguageModel slice out of the given config
Expand Down
6 changes: 3 additions & 3 deletions pkg/routers/health/buckets_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,17 +47,17 @@ func TestTokenBucket_TakeConcurrently(t *testing.T) {

wg.Wait()

if time.Now().Sub(before) < 1*time.Second {
if time.Since(before) < 1*time.Second {
t.Fatal("Did not wait 1s")
}
}

func TestTokenBucket_TokenNumberIsCorrect(t *testing.T) {
bucket := NewTokenBucket(1, 10)
require.Equal(t, 10.0, bucket.Tokens())
require.Equal(t, 10, bucket.Tokens())

require.NoError(t, bucket.Take(2))
require.InEpsilon(t, 8.0, bucket.Tokens(), 0.0001)
require.InEpsilon(t, 8, bucket.Tokens(), 0.0001)

require.NoError(t, bucket.Take(2))
require.InEpsilon(t, 6.0, bucket.Tokens(), 0.0001)
Expand Down
6 changes: 3 additions & 3 deletions pkg/routers/retry/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,9 @@ package retry
import "time"

type ExpRetryConfig struct {
MaxRetries int
MinDelay time.Duration
MaxDelay *time.Duration
MaxRetries int `yaml:"max_retries,omitempty" json:"max_retries"`
MinDelay time.Duration `yaml:"min_delay,omitempty" json:"min_delay" swaggertype:"primitive,integer"`
MaxDelay *time.Duration `yaml:"max_delay,omitempty" json:"max_delay" swaggertype:"primitive,integer"`
}

func DefaultExpRetryConfig() *ExpRetryConfig {
Expand Down

0 comments on commit 9008a33

Please sign in to comment.