diff --git a/cacheservice/cmd/entrypoints/migrate.go b/cacheservice/cmd/entrypoints/migrate.go index a4ce8b1b48..bc6d537655 100644 --- a/cacheservice/cmd/entrypoints/migrate.go +++ b/cacheservice/cmd/entrypoints/migrate.go @@ -1,7 +1,11 @@ package entrypoints import ( + "context" + "github.com/spf13/cobra" + + "github.com/flyteorg/flyte/cacheservice/pkg/repositories" ) var parentMigrateCmd = &cobra.Command{ @@ -9,12 +13,13 @@ var parentMigrateCmd = &cobra.Command{ Short: "This command controls migration behavior for the Flyte cacheservice database. Please choose a subcommand.", } -// This runs all the migrations. This is a placeholder for now as cache service does not have any migrations +// This runs all the migrations for sql databases var migrateCmd = &cobra.Command{ Use: "run", Short: "This command will run all the migrations for the database", RunE: func(cmd *cobra.Command, args []string) error { - return nil + ctx := context.Background() + return repositories.Migrate(ctx) }, } diff --git a/cacheservice/go.mod b/cacheservice/go.mod index 34bc9af6c2..1936b87785 100644 --- a/cacheservice/go.mod +++ b/cacheservice/go.mod @@ -3,6 +3,7 @@ module github.com/flyteorg/flyte/cacheservice go 1.21 require ( + github.com/Selvatico/go-mocket v1.0.7 github.com/aws/aws-sdk-go-v2/config v1.26.6 github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.12.17 github.com/flyteorg/flyte/flyteidl v0.0.0-00010101000000-000000000000 @@ -10,6 +11,7 @@ require ( github.com/go-redis/redis/v8 v8.11.5 github.com/golang/glog v1.1.2 github.com/golang/protobuf v1.5.3 + github.com/jackc/pgconn v1.14.1 github.com/mitchellh/mapstructure v1.5.0 github.com/spf13/cobra v1.7.0 github.com/spf13/pflag v1.0.5 @@ -17,6 +19,8 @@ require ( go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.47.0 go.opentelemetry.io/otel v1.22.0 google.golang.org/grpc v1.60.1 + gorm.io/driver/postgres v1.5.3 + gorm.io/gorm v1.25.4 ) require ( @@ -34,8 +38,19 @@ require ( github.com/aws/aws-sdk-go-v2/service/sts v1.26.7 // indirect github.com/aws/smithy-go v1.19.0 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect + github.com/go-gormigrate/gormigrate/v2 v2.1.1 // indirect github.com/google/s2a-go v0.1.4 // indirect + github.com/jackc/chunkreader/v2 v2.0.1 // indirect + github.com/jackc/pgio v1.0.0 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgproto3/v2 v2.3.2 // indirect + github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect + github.com/jackc/pgx/v5 v5.4.3 // indirect + github.com/jinzhu/inflection v1.0.0 // indirect + github.com/jinzhu/now v1.1.5 // indirect + github.com/mattn/go-sqlite3 v1.14.17 // indirect golang.org/x/sync v0.4.0 // indirect + gorm.io/driver/sqlite v1.5.4 // indirect ) require ( diff --git a/cacheservice/go.sum b/cacheservice/go.sum index bf6e90320b..b580301548 100644 --- a/cacheservice/go.sum +++ b/cacheservice/go.sum @@ -62,6 +62,8 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/Selvatico/go-mocket v1.0.7 h1:sXuFMnMfVL9b/Os8rGXPgbOFbr4HJm8aHsulD/uMTUk= +github.com/Selvatico/go-mocket v1.0.7/go.mod h1:4gO2v+uQmsL+jzQgLANy3tyEFzaEzHlymVbZ3GP2Oes= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/aws/aws-sdk-go v1.44.2 h1:5VBk5r06bgxgRKVaUtm1/4NT/rtrnH2E4cnAYv5zgQc= github.com/aws/aws-sdk-go v1.44.2/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= @@ -122,9 +124,13 @@ github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWH github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4 h1:/inchEIKaYC1Akx+H+gqO04wryn5h75LSazbRlnya1k= github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= github.com/coocood/freecache v1.1.1 h1:uukNF7QKCZEdZ9gAV7WQzvh0SbjwdMF6m3x3rxEkaPc= github.com/coocood/freecache v1.1.1/go.mod h1:OKrEjkGVoxZhyWAJoeFi5BMLUJm2Tit0kpGkIr7NGYY= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -157,6 +163,8 @@ github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeME github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gormigrate/gormigrate/v2 v2.1.1 h1:eGS0WTFRV30r103lU8JNXY27KbviRnqqIDobW3EV3iY= +github.com/go-gormigrate/gormigrate/v2 v2.1.1/go.mod h1:L7nJ620PFDKei9QOhJzqA8kRCk+E3UbV2f5gv+1ndLc= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= @@ -173,6 +181,7 @@ github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/ github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI= github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/go-test/deep v1.0.7 h1:/VSMRlnY/JSyqxQUzQLKVMAskpY/NZKFA5j2P+0pP2M= @@ -276,7 +285,52 @@ github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28= github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo= +github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8= +github.com/jackc/chunkreader/v2 v2.0.1/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/pgconn v0.0.0-20190420214824-7e0022ef6ba3/go.mod h1:jkELnwuX+w9qN5YIfX0fl88Ehu4XC3keFuOJJk9pcnA= +github.com/jackc/pgconn v0.0.0-20190824142844-760dd75542eb/go.mod h1:lLjNuW/+OfW9/pnVKPazfWOgNfH2aPem8YQ7ilXGvJE= +github.com/jackc/pgconn v0.0.0-20190831204454-2fabfa3c18b7/go.mod h1:ZJKsE/KZfsUgOEh9hBm+xYTstcNHg7UPMVJqRfQxq4s= +github.com/jackc/pgconn v1.8.0/go.mod h1:1C2Pb36bGIP9QHGBYCjnyhqu7Rv3sGshaQUvmfGIB/o= +github.com/jackc/pgconn v1.9.0/go.mod h1:YctiPyvzfU11JFxoXokUOOKQXQmDMoJL9vJzHH8/2JY= +github.com/jackc/pgconn v1.14.1 h1:smbxIaZA08n6YuxEX1sDyjV/qkbtUtkH20qLkR9MUR4= +github.com/jackc/pgconn v1.14.1/go.mod h1:9mBNlny0UvkgJdCDvdVHYSjI+8tD2rnKK69Wz8ti++E= +github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= +github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= +github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE= +github.com/jackc/pgmock v0.0.0-20201204152224-4fe30f7445fd/go.mod h1:hrBW0Enj2AZTNpt/7Y5rr2xe/9Mn757Wtb2xeBzPv2c= +github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65 h1:DadwsjnMwFjfWc9y5Wi/+Zz7xoE5ALHsRQlOctkOiHc= +github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65/go.mod h1:5R2h2EEX+qri8jOWMbJCtaPWkrrNc7OHwsp2TCqp7ak= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgproto3 v1.1.0/go.mod h1:eR5FA3leWg7p9aeAqi37XOTgTIbkABlvcPB3E5rlc78= +github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190420180111-c116219b62db/go.mod h1:bhq50y+xrl9n5mRYyCBFKkpRVTLYJVWeCc+mEAI3yXA= +github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190609003834-432c2951c711/go.mod h1:uH0AWtUmuShn0bcesswc4aBTWGvw0cAxIJp+6OB//Wg= +github.com/jackc/pgproto3/v2 v2.0.0-rc3/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= +github.com/jackc/pgproto3/v2 v2.0.0-rc3.0.20190831210041-4c03ce451f29/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= +github.com/jackc/pgproto3/v2 v2.0.6/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgproto3/v2 v2.1.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgproto3/v2 v2.3.2 h1:7eY55bdBeCz1F2fTzSz69QC+pG46jYq9/jtSPiJ5nn0= +github.com/jackc/pgproto3/v2 v2.3.2/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01CGwFsrv11mJRHWJ6aifDLfdV3aVjFF0zg= +github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc= +github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw= +github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y= +github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM= +github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc= +github.com/jackc/pgx/v5 v5.4.3 h1:cxFyXhxlvAifxnkKKdlxv8XqUf59tDlYjnV5YYfsJJY= +github.com/jackc/pgx/v5 v5.4.3/go.mod h1:Ig06C2Vu0t5qXC60W8sqIthScaEnFvojjj9dSljmHRA= +github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= +github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= +github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= +github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= @@ -289,27 +343,38 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/magiconair/properties v1.8.6 h1:5ibWZ6iY0NctNGWo87LalDlEZ6R41TqbbDamhfG/Qzo= github.com/magiconair/properties v1.8.6/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40= github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= +github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/mattn/go-sqlite3 v1.14.17 h1:mCRHCLDUBXgpKAqIKsaAaAsrAlbkeomtRFKXh2L6YIM= +github.com/mattn/go-sqlite3 v1.14.17/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= @@ -356,7 +421,14 @@ github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6L github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= +github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= +github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= +github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= +github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= +github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.9.2 h1:oxx1eChJGI6Uks2ZC4W1zpLlVgqB8ner4EuQwV4Ik1Y= github.com/sirupsen/logrus v1.9.2/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ= @@ -374,6 +446,8 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An github.com/spf13/viper v1.11.0 h1:7OX/1FS6n7jHD1zGrZTM7WtY13ZELRyosK4k93oPr44= github.com/spf13/viper v1.11.0/go.mod h1:djo0X/bA5+tYVoCn+C7cAYJGcVn/qYLFTG8gdUsX7Zk= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= @@ -394,6 +468,7 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -417,19 +492,30 @@ go.opentelemetry.io/otel/sdk v1.22.0/go.mod h1:iu7luyVGYovrRpe2fmj3CVKouQNdTOkxt go.opentelemetry.io/otel/trace v1.22.0 h1:Hg6pPujv0XG9QaVbGOBVHunyuLcCC3jN7WEhPx83XD0= go.opentelemetry.io/otel/trace v1.22.0/go.mod h1:RbbHXVqKES9QhzZq/fE5UnOSILqRt40a21sPw2He1xo= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= +go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.25.0 h1:4Hvk6GtkucQ790dqmj7l1eEnRdKm3k3ZUrUMS2d5+5c= go.uber.org/zap v1.25.0/go.mod h1:JIAUzQIH94IC4fOJQm7gMmBJP5k7wQfdcnYdPoEXJYk= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc= golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -480,6 +566,7 @@ golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -503,6 +590,7 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo= golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -530,15 +618,21 @@ golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.4.0 h1:zxkM55ReGkDlKSM+Fu41A+zmbZuaPVbGMzvvdUPznYQ= golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -573,10 +667,13 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.16.0 h1:m+B6fahuftsE9qjo0VWp2FW0mB3MTJvR0BaMQrq0pmE= golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -588,6 +685,7 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -602,12 +700,14 @@ golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190823170909-c4a336ef6a2f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -647,6 +747,8 @@ golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.9.3 h1:Gn1I8+64MsuTb/HpH+LmQtNas23LhUVr3rYZ0eKuaMM= golang.org/x/tools v0.9.3/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= +golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -768,6 +870,7 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.66.4 h1:SsAcf+mM7mRZo2nJNGt8mZCjG8ZRaNGMURJw7BsIST4= @@ -782,6 +885,12 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gorm.io/driver/postgres v1.5.3 h1:qKGY5CPHOuj47K/VxbCXJfFvIUeqMSXXadqdCY+MbBU= +gorm.io/driver/postgres v1.5.3/go.mod h1:F+LtvlFhZT7UBiA81mC9W6Su3D4WUhSboc/36QZU0gk= +gorm.io/driver/sqlite v1.5.4 h1:IqXwXi8M/ZlPzH/947tn5uik3aYQslP9BVveoax0nV0= +gorm.io/driver/sqlite v1.5.4/go.mod h1:qxAuCol+2r6PannQDpOP1FP6ag3mKi4esLnB/jHed+4= +gorm.io/gorm v1.25.4 h1:iyNd8fNAe8W9dvtlgeRI5zSVZPsq3OpcTu37cYcpCmw= +gorm.io/gorm v1.25.4/go.mod h1:L4uxeKpfBml98NYqVqwAdmV1a2nBtAec/cf3fpucW/k= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/cacheservice/pkg/manager/impl/cache_data_store_test.go b/cacheservice/pkg/manager/impl/cache_data_store_test.go deleted file mode 100644 index 4f9d22e382..0000000000 --- a/cacheservice/pkg/manager/impl/cache_data_store_test.go +++ /dev/null @@ -1 +0,0 @@ -package impl diff --git a/cacheservice/pkg/manager/impl/cache_manager.go b/cacheservice/pkg/manager/impl/cache_manager.go index eb30bc795b..82ea9a5cca 100644 --- a/cacheservice/pkg/manager/impl/cache_manager.go +++ b/cacheservice/pkg/manager/impl/cache_manager.go @@ -12,8 +12,9 @@ import ( "github.com/flyteorg/flyte/cacheservice/pkg/errors" "github.com/flyteorg/flyte/cacheservice/pkg/manager/impl/validators" "github.com/flyteorg/flyte/cacheservice/pkg/manager/interfaces" - "github.com/flyteorg/flyte/cacheservice/repositories/models" - "github.com/flyteorg/flyte/cacheservice/repositories/transformers" + repoInterfaces "github.com/flyteorg/flyte/cacheservice/pkg/repositories/interfaces" + "github.com/flyteorg/flyte/cacheservice/pkg/repositories/models" + "github.com/flyteorg/flyte/cacheservice/pkg/repositories/transformers" "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/cacheservice" "github.com/flyteorg/flyte/flytestdlib/logger" "github.com/flyteorg/flyte/flytestdlib/promutils" @@ -49,8 +50,8 @@ type cacheMetrics struct { type cacheManager struct { outputStore interfaces.CacheOutputBlobStore - dataStore interfaces.CacheDataStoreClient - reservationStore interfaces.ReservationDataStoreClient + dataStore repoInterfaces.CachedOutputRepo + reservationStore repoInterfaces.ReservationRepo systemMetrics cacheMetrics maxInlineSizeBytes int64 heartbeatGracePeriodMultiplier time.Duration @@ -111,6 +112,7 @@ func (m *cacheManager) Put(ctx context.Context, request *cacheservice.PutCacheRe return nil, errors.NewCacheServiceErrorf(codes.Internal, "Failed to create cached output model, err: %v", err) } + // TODO - @pvditt - can do this in single transaction w/ postgres client - move logic to client (still need to delete blob) cachedOutput, err := m.dataStore.Get(ctx, request.Key) var notFound bool if err != nil { @@ -208,7 +210,7 @@ func (m *cacheManager) GetOrExtendReservation(ctx context.Context, request *cach heartbeatInterval = request.GetHeartbeatInterval().AsDuration() } - newReservation := &models.Reservation{ + newReservation := &models.CacheReservation{ Key: resKey, OwnerID: request.OwnerId, ExpiresAt: now.Add(heartbeatInterval * m.heartbeatGracePeriodMultiplier), @@ -219,7 +221,7 @@ func (m *cacheManager) GetOrExtendReservation(ctx context.Context, request *cach if reservationModel.ExpiresAt.Before(now) || reservationModel.OwnerID == request.OwnerId { storeError = m.reservationStore.Update(ctx, newReservation, now) } else { - logger.Debugf(ctx, "Reservation: %+v is held by %s", reservationModel.Key, reservationModel.OwnerID) + logger.Debugf(ctx, "CacheReservation: %+v is held by %s", reservationModel.Key, reservationModel.OwnerID) reservation := transformers.FromReservationModel(ctx, reservationModel) return &cacheservice.GetOrExtendReservationResponse{Reservation: reservation}, nil } @@ -229,7 +231,7 @@ func (m *cacheManager) GetOrExtendReservation(ctx context.Context, request *cach if storeError != nil { if status.Code(storeError) == codes.AlreadyExists { - logger.Debugf(ctx, "Reservation: %+v already exists", newReservation.Key) + logger.Debugf(ctx, "CacheReservation: %+v already exists", newReservation.Key) newReservation, err = m.reservationStore.Get(ctx, resKey) if err != nil { logger.Errorf(ctx, "Failed to Get reservation in reservation store, err: %v", err) @@ -265,7 +267,7 @@ func (m *cacheManager) ReleaseReservation(ctx context.Context, request *cacheser err = m.reservationStore.Delete(ctx, resKey, request.OwnerId) if err != nil { if status.Code(err) == codes.NotFound { - logger.Debugf(ctx, "Reservation with key %v and owner %v not found", request.Key, request.OwnerId) + logger.Debugf(ctx, "CacheReservation with key %v and owner %v not found", request.Key, request.OwnerId) m.systemMetrics.notFoundCounter.Inc(ctx) return &cacheservice.ReleaseReservationResponse{}, nil } @@ -278,7 +280,7 @@ func (m *cacheManager) ReleaseReservation(ctx context.Context, request *cacheser return &cacheservice.ReleaseReservationResponse{}, nil } -func NewCacheManager(outputStore interfaces.CacheOutputBlobStore, dataStore interfaces.CacheDataStoreClient, reservationStore interfaces.ReservationDataStoreClient, maxInlineSizeBytes int64, cacheScope promutils.Scope, +func NewCacheManager(outputStore interfaces.CacheOutputBlobStore, dataStore repoInterfaces.CachedOutputRepo, reservationStore repoInterfaces.ReservationRepo, maxInlineSizeBytes int64, cacheScope promutils.Scope, heartbeatGracePeriodMultiplier time.Duration, maxHeartbeatInterval time.Duration) interfaces.CacheManager { cacheMetrics := cacheMetrics{ scope: cacheScope, diff --git a/cacheservice/pkg/manager/impl/cache_manager_test.go b/cacheservice/pkg/manager/impl/cache_manager_test.go index 1ca6d6b5e7..f8eba62bda 100644 --- a/cacheservice/pkg/manager/impl/cache_manager_test.go +++ b/cacheservice/pkg/manager/impl/cache_manager_test.go @@ -15,7 +15,8 @@ import ( "github.com/flyteorg/flyte/cacheservice/pkg/errors" "github.com/flyteorg/flyte/cacheservice/pkg/manager/interfaces" "github.com/flyteorg/flyte/cacheservice/pkg/manager/mocks" - "github.com/flyteorg/flyte/cacheservice/repositories/models" + repoMocks "github.com/flyteorg/flyte/cacheservice/pkg/repositories/mocks" + "github.com/flyteorg/flyte/cacheservice/pkg/repositories/models" "github.com/flyteorg/flyte/flyteidl/clients/go/coreutils" "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/cacheservice" "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/core" @@ -91,7 +92,7 @@ func TestCacheManager_Get(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - mockDataStore := &mocks.CacheDataStoreClient{} + mockDataStore := &repoMocks.CachedOutputRepo{} mockDataStore.OnGetMatch( ctx, mock.MatchedBy(func(o string) bool { @@ -99,7 +100,7 @@ func TestCacheManager_Get(t *testing.T) { return true })).Return(tc.mockDataStoreReturn, tc.mockDataStoreError) - m := NewCacheManager(&mocks.CacheOutputBlobStore{}, mockDataStore, &mocks.ReservationDataStoreClient{}, 1024, mockScope.NewTestScope(), time.Duration(1), time.Duration(1)) + m := NewCacheManager(&mocks.CacheOutputBlobStore{}, mockDataStore, &repoMocks.ReservationRepo{}, 1024, mockScope.NewTestScope(), time.Duration(1), time.Duration(1)) response, err := m.Get(ctx, tc.mockRequest) @@ -373,7 +374,7 @@ func TestCacheManager_Put(t *testing.T) { } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - mockDataStore := &mocks.CacheDataStoreClient{} + mockDataStore := &repoMocks.CachedOutputRepo{} mockDataStore.OnGetMatch( ctx, mock.MatchedBy(matchKeyFunc), @@ -395,7 +396,7 @@ func TestCacheManager_Put(t *testing.T) { mock.MatchedBy(tc.outputDeleteMatchOutputFunc), ).Return(tc.outputDeleteError) - m := NewCacheManager(mockOutputStore, mockDataStore, &mocks.ReservationDataStoreClient{}, tc.maxSizeBytes, mockScope.NewTestScope(), time.Duration(1), time.Duration(1)) + m := NewCacheManager(mockOutputStore, mockDataStore, &repoMocks.ReservationRepo{}, tc.maxSizeBytes, mockScope.NewTestScope(), time.Duration(1), time.Duration(1)) _, err := m.Put(ctx, tc.mockRequest) @@ -431,9 +432,9 @@ func TestCacheManager_GetOrExtendReservation(t *testing.T) { expectError bool expectedErrorStatusCode codes.Code deleteError error - getReturn *models.Reservation + getReturn *models.CacheReservation getError error - getReturn1 *models.Reservation + getReturn1 *models.CacheReservation getError1 error createError error updateError error @@ -499,7 +500,7 @@ func TestCacheManager_GetOrExtendReservation(t *testing.T) { }, getReturn: nil, getError: errors.NewNotFoundError("reservation", sampleKey), - getReturn1: &models.Reservation{ + getReturn1: &models.CacheReservation{ Key: sampleKey, OwnerID: sampleOwner1, ExpiresAt: now.Add(time.Hour), @@ -517,7 +518,7 @@ func TestCacheManager_GetOrExtendReservation(t *testing.T) { OwnerId: sampleOwner, HeartbeatInterval: sampleHeartBeatInterval, }, - getReturn: &models.Reservation{ + getReturn: &models.CacheReservation{ Key: sampleKey, OwnerID: sampleOwner1, ExpiresAt: now.Add(-time.Hour), @@ -535,7 +536,7 @@ func TestCacheManager_GetOrExtendReservation(t *testing.T) { OwnerId: sampleOwner, HeartbeatInterval: sampleHeartBeatInterval, }, - getReturn: &models.Reservation{ + getReturn: &models.CacheReservation{ Key: sampleKey, OwnerID: sampleOwner, ExpiresAt: now.Add(time.Hour), @@ -553,7 +554,7 @@ func TestCacheManager_GetOrExtendReservation(t *testing.T) { OwnerId: sampleOwner, HeartbeatInterval: sampleHeartBeatInterval, }, - getReturn: &models.Reservation{ + getReturn: &models.CacheReservation{ Key: sampleKey, OwnerID: sampleOwner1, ExpiresAt: now.Add(time.Hour), @@ -570,7 +571,7 @@ func TestCacheManager_GetOrExtendReservation(t *testing.T) { for _, tc := range testCases { requestKey := fmt.Sprintf("%s:%s", "reservation", tc.mockRequest.Key) t.Run(tc.name, func(t *testing.T) { - mockReservationStoreClient := &mocks.ReservationDataStoreClient{} + mockReservationStoreClient := &repoMocks.ReservationRepo{} mockReservationStoreClient.OnGetMatch( ctx, mock.MatchedBy(func(o string) bool { @@ -585,7 +586,7 @@ func TestCacheManager_GetOrExtendReservation(t *testing.T) { })).Return(tc.getReturn1, tc.getError1) mockReservationStoreClient.OnCreateMatch( ctx, - mock.MatchedBy(func(reservation *models.Reservation) bool { + mock.MatchedBy(func(reservation *models.CacheReservation) bool { assert.Equal(t, requestKey, reservation.Key) assert.Equal(t, tc.expectedExpiresAtCall, reservation.ExpiresAt) return true @@ -594,7 +595,7 @@ func TestCacheManager_GetOrExtendReservation(t *testing.T) { ).Return(tc.createError) mockReservationStoreClient.OnUpdateMatch( ctx, - mock.MatchedBy(func(reservation *models.Reservation) bool { + mock.MatchedBy(func(reservation *models.CacheReservation) bool { assert.Equal(t, requestKey, reservation.Key) assert.Equal(t, tc.expectedExpiresAtCall, reservation.ExpiresAt) return true @@ -602,7 +603,7 @@ func TestCacheManager_GetOrExtendReservation(t *testing.T) { mock.Anything, ).Return(tc.updateError) - m := NewCacheManager(&mocks.CacheOutputBlobStore{}, &mocks.CacheDataStoreClient{}, mockReservationStoreClient, 0, mockScope.NewTestScope(), heartbeatGracePeriodMultiplier, maxHeartBeatInterval) + m := NewCacheManager(&mocks.CacheOutputBlobStore{}, &repoMocks.CachedOutputRepo{}, mockReservationStoreClient, 0, mockScope.NewTestScope(), heartbeatGracePeriodMultiplier, maxHeartBeatInterval) reservation, err := m.GetOrExtendReservation(ctx, tc.mockRequest, now) if tc.expectError { @@ -680,7 +681,7 @@ func TestCacheManager_ReleaseReservation(t *testing.T) { t.Run(tc.name, func(t *testing.T) { requestKey := fmt.Sprintf("%s:%s", "reservation", tc.mockRequest.Key) - mockReservationStoreClient := &mocks.ReservationDataStoreClient{} + mockReservationStoreClient := &repoMocks.ReservationRepo{} mockReservationStoreClient.OnDeleteMatch( ctx, mock.MatchedBy(func(key string) bool { @@ -692,7 +693,7 @@ func TestCacheManager_ReleaseReservation(t *testing.T) { }), ).Return(tc.deleteError) - m := NewCacheManager(&mocks.CacheOutputBlobStore{}, &mocks.CacheDataStoreClient{}, mockReservationStoreClient, 0, mockScope.NewTestScope(), time.Duration(1), time.Duration(1)) + m := NewCacheManager(&mocks.CacheOutputBlobStore{}, &repoMocks.CachedOutputRepo{}, mockReservationStoreClient, 0, mockScope.NewTestScope(), time.Duration(1), time.Duration(1)) reservation, err := m.ReleaseReservation(ctx, tc.mockRequest) if tc.expectError { diff --git a/cacheservice/pkg/manager/impl/cache_output_blob_store.go b/cacheservice/pkg/manager/impl/cache_output_blob_store.go index ab514ed725..228f715ceb 100644 --- a/cacheservice/pkg/manager/impl/cache_output_blob_store.go +++ b/cacheservice/pkg/manager/impl/cache_output_blob_store.go @@ -45,7 +45,7 @@ func (m *cacheOutputBlobStore) Delete(ctx context.Context, uri string) error { return nil } -func NewCacheOutputStore(store *storage.DataStore, storagePrefix storage.DataReference) interfaces.CacheOutputBlobStore { +func NewCacheOutputBlobStore(store *storage.DataStore, storagePrefix storage.DataReference) interfaces.CacheOutputBlobStore { return &cacheOutputBlobStore{ store: store, storagePrefix: storagePrefix, diff --git a/cacheservice/pkg/manager/impl/reservation_store_test.go b/cacheservice/pkg/manager/impl/reservation_store_test.go deleted file mode 100644 index 4f9d22e382..0000000000 --- a/cacheservice/pkg/manager/impl/reservation_store_test.go +++ /dev/null @@ -1 +0,0 @@ -package impl diff --git a/cacheservice/pkg/manager/interfaces/cache.go b/cacheservice/pkg/manager/interfaces/cache.go index 28d275d15e..3968d11c9c 100644 --- a/cacheservice/pkg/manager/interfaces/cache.go +++ b/cacheservice/pkg/manager/interfaces/cache.go @@ -4,7 +4,6 @@ import ( "context" "time" - "github.com/flyteorg/flyte/cacheservice/repositories/models" "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/cacheservice" "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/core" ) @@ -23,16 +22,3 @@ type CacheOutputBlobStore interface { Create(ctx context.Context, key string, output *core.LiteralMap) (string, error) Delete(ctx context.Context, uri string) error } - -type CacheDataStoreClient interface { - Get(ctx context.Context, key string) (*models.CachedOutput, error) - Put(ctx context.Context, key string, cachedOutput *models.CachedOutput) error - Delete(ctx context.Context, key string) error -} - -type ReservationDataStoreClient interface { - Create(ctx context.Context, reservation *models.Reservation, now time.Time) error - Update(ctx context.Context, reservation *models.Reservation, now time.Time) error - Get(ctx context.Context, key string) (*models.Reservation, error) - Delete(ctx context.Context, key string, ownerID string) error -} diff --git a/cacheservice/pkg/manager/mocks/cache_data_store_client.go b/cacheservice/pkg/manager/mocks/cache_data_store_client.go deleted file mode 100644 index d97175ed57..0000000000 --- a/cacheservice/pkg/manager/mocks/cache_data_store_client.go +++ /dev/null @@ -1,121 +0,0 @@ -// Code generated by mockery v1.0.1. DO NOT EDIT. - -package mocks - -import ( - context "context" - - mock "github.com/stretchr/testify/mock" - - models "github.com/flyteorg/flyte/cacheservice/repositories/models" -) - -// CacheDataStoreClient is an autogenerated mock type for the CacheDataStoreClient type -type CacheDataStoreClient struct { - mock.Mock -} - -type CacheDataStoreClient_Delete struct { - *mock.Call -} - -func (_m CacheDataStoreClient_Delete) Return(_a0 error) *CacheDataStoreClient_Delete { - return &CacheDataStoreClient_Delete{Call: _m.Call.Return(_a0)} -} - -func (_m *CacheDataStoreClient) OnDelete(ctx context.Context, key string) *CacheDataStoreClient_Delete { - c_call := _m.On("Delete", ctx, key) - return &CacheDataStoreClient_Delete{Call: c_call} -} - -func (_m *CacheDataStoreClient) OnDeleteMatch(matchers ...interface{}) *CacheDataStoreClient_Delete { - c_call := _m.On("Delete", matchers...) - return &CacheDataStoreClient_Delete{Call: c_call} -} - -// Delete provides a mock function with given fields: ctx, key -func (_m *CacheDataStoreClient) Delete(ctx context.Context, key string) error { - ret := _m.Called(ctx, key) - - var r0 error - if rf, ok := ret.Get(0).(func(context.Context, string) error); ok { - r0 = rf(ctx, key) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -type CacheDataStoreClient_Get struct { - *mock.Call -} - -func (_m CacheDataStoreClient_Get) Return(_a0 *models.CachedOutput, _a1 error) *CacheDataStoreClient_Get { - return &CacheDataStoreClient_Get{Call: _m.Call.Return(_a0, _a1)} -} - -func (_m *CacheDataStoreClient) OnGet(ctx context.Context, key string) *CacheDataStoreClient_Get { - c_call := _m.On("Get", ctx, key) - return &CacheDataStoreClient_Get{Call: c_call} -} - -func (_m *CacheDataStoreClient) OnGetMatch(matchers ...interface{}) *CacheDataStoreClient_Get { - c_call := _m.On("Get", matchers...) - return &CacheDataStoreClient_Get{Call: c_call} -} - -// Get provides a mock function with given fields: ctx, key -func (_m *CacheDataStoreClient) Get(ctx context.Context, key string) (*models.CachedOutput, error) { - ret := _m.Called(ctx, key) - - var r0 *models.CachedOutput - if rf, ok := ret.Get(0).(func(context.Context, string) *models.CachedOutput); ok { - r0 = rf(ctx, key) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(*models.CachedOutput) - } - } - - var r1 error - if rf, ok := ret.Get(1).(func(context.Context, string) error); ok { - r1 = rf(ctx, key) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -type CacheDataStoreClient_Put struct { - *mock.Call -} - -func (_m CacheDataStoreClient_Put) Return(_a0 error) *CacheDataStoreClient_Put { - return &CacheDataStoreClient_Put{Call: _m.Call.Return(_a0)} -} - -func (_m *CacheDataStoreClient) OnPut(ctx context.Context, key string, cachedOutput *models.CachedOutput) *CacheDataStoreClient_Put { - c_call := _m.On("Put", ctx, key, cachedOutput) - return &CacheDataStoreClient_Put{Call: c_call} -} - -func (_m *CacheDataStoreClient) OnPutMatch(matchers ...interface{}) *CacheDataStoreClient_Put { - c_call := _m.On("Put", matchers...) - return &CacheDataStoreClient_Put{Call: c_call} -} - -// Put provides a mock function with given fields: ctx, key, cachedOutput -func (_m *CacheDataStoreClient) Put(ctx context.Context, key string, cachedOutput *models.CachedOutput) error { - ret := _m.Called(ctx, key, cachedOutput) - - var r0 error - if rf, ok := ret.Get(0).(func(context.Context, string, *models.CachedOutput) error); ok { - r0 = rf(ctx, key, cachedOutput) - } else { - r0 = ret.Error(0) - } - - return r0 -} diff --git a/cacheservice/pkg/manager/mocks/reservation_data_store_client.go b/cacheservice/pkg/manager/mocks/reservation_data_store_client.go deleted file mode 100644 index 75751b6901..0000000000 --- a/cacheservice/pkg/manager/mocks/reservation_data_store_client.go +++ /dev/null @@ -1,155 +0,0 @@ -// Code generated by mockery v1.0.1. DO NOT EDIT. - -package mocks - -import ( - context "context" - - mock "github.com/stretchr/testify/mock" - - models "github.com/flyteorg/flyte/cacheservice/repositories/models" - - time "time" -) - -// ReservationDataStoreClient is an autogenerated mock type for the ReservationDataStoreClient type -type ReservationDataStoreClient struct { - mock.Mock -} - -type ReservationDataStoreClient_Create struct { - *mock.Call -} - -func (_m ReservationDataStoreClient_Create) Return(_a0 error) *ReservationDataStoreClient_Create { - return &ReservationDataStoreClient_Create{Call: _m.Call.Return(_a0)} -} - -func (_m *ReservationDataStoreClient) OnCreate(ctx context.Context, reservation *models.Reservation, now time.Time) *ReservationDataStoreClient_Create { - c_call := _m.On("Create", ctx, reservation, now) - return &ReservationDataStoreClient_Create{Call: c_call} -} - -func (_m *ReservationDataStoreClient) OnCreateMatch(matchers ...interface{}) *ReservationDataStoreClient_Create { - c_call := _m.On("Create", matchers...) - return &ReservationDataStoreClient_Create{Call: c_call} -} - -// Create provides a mock function with given fields: ctx, reservation, now -func (_m *ReservationDataStoreClient) Create(ctx context.Context, reservation *models.Reservation, now time.Time) error { - ret := _m.Called(ctx, reservation, now) - - var r0 error - if rf, ok := ret.Get(0).(func(context.Context, *models.Reservation, time.Time) error); ok { - r0 = rf(ctx, reservation, now) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -type ReservationDataStoreClient_Delete struct { - *mock.Call -} - -func (_m ReservationDataStoreClient_Delete) Return(_a0 error) *ReservationDataStoreClient_Delete { - return &ReservationDataStoreClient_Delete{Call: _m.Call.Return(_a0)} -} - -func (_m *ReservationDataStoreClient) OnDelete(ctx context.Context, key string, ownerID string) *ReservationDataStoreClient_Delete { - c_call := _m.On("Delete", ctx, key, ownerID) - return &ReservationDataStoreClient_Delete{Call: c_call} -} - -func (_m *ReservationDataStoreClient) OnDeleteMatch(matchers ...interface{}) *ReservationDataStoreClient_Delete { - c_call := _m.On("Delete", matchers...) - return &ReservationDataStoreClient_Delete{Call: c_call} -} - -// Delete provides a mock function with given fields: ctx, key, ownerID -func (_m *ReservationDataStoreClient) Delete(ctx context.Context, key string, ownerID string) error { - ret := _m.Called(ctx, key, ownerID) - - var r0 error - if rf, ok := ret.Get(0).(func(context.Context, string, string) error); ok { - r0 = rf(ctx, key, ownerID) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -type ReservationDataStoreClient_Get struct { - *mock.Call -} - -func (_m ReservationDataStoreClient_Get) Return(_a0 *models.Reservation, _a1 error) *ReservationDataStoreClient_Get { - return &ReservationDataStoreClient_Get{Call: _m.Call.Return(_a0, _a1)} -} - -func (_m *ReservationDataStoreClient) OnGet(ctx context.Context, key string) *ReservationDataStoreClient_Get { - c_call := _m.On("Get", ctx, key) - return &ReservationDataStoreClient_Get{Call: c_call} -} - -func (_m *ReservationDataStoreClient) OnGetMatch(matchers ...interface{}) *ReservationDataStoreClient_Get { - c_call := _m.On("Get", matchers...) - return &ReservationDataStoreClient_Get{Call: c_call} -} - -// Get provides a mock function with given fields: ctx, key -func (_m *ReservationDataStoreClient) Get(ctx context.Context, key string) (*models.Reservation, error) { - ret := _m.Called(ctx, key) - - var r0 *models.Reservation - if rf, ok := ret.Get(0).(func(context.Context, string) *models.Reservation); ok { - r0 = rf(ctx, key) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(*models.Reservation) - } - } - - var r1 error - if rf, ok := ret.Get(1).(func(context.Context, string) error); ok { - r1 = rf(ctx, key) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -type ReservationDataStoreClient_Update struct { - *mock.Call -} - -func (_m ReservationDataStoreClient_Update) Return(_a0 error) *ReservationDataStoreClient_Update { - return &ReservationDataStoreClient_Update{Call: _m.Call.Return(_a0)} -} - -func (_m *ReservationDataStoreClient) OnUpdate(ctx context.Context, reservation *models.Reservation, now time.Time) *ReservationDataStoreClient_Update { - c_call := _m.On("Update", ctx, reservation, now) - return &ReservationDataStoreClient_Update{Call: c_call} -} - -func (_m *ReservationDataStoreClient) OnUpdateMatch(matchers ...interface{}) *ReservationDataStoreClient_Update { - c_call := _m.On("Update", matchers...) - return &ReservationDataStoreClient_Update{Call: c_call} -} - -// Update provides a mock function with given fields: ctx, reservation, now -func (_m *ReservationDataStoreClient) Update(ctx context.Context, reservation *models.Reservation, now time.Time) error { - ret := _m.Called(ctx, reservation, now) - - var r0 error - if rf, ok := ret.Get(0).(func(context.Context, *models.Reservation, time.Time) error); ok { - r0 = rf(ctx, reservation, now) - } else { - r0 = ret.Error(0) - } - - return r0 -} diff --git a/cacheservice/pkg/manager/impl/cache_data_store.go b/cacheservice/pkg/repositories/clients/cache_data_store.go similarity index 71% rename from cacheservice/pkg/manager/impl/cache_data_store.go rename to cacheservice/pkg/repositories/clients/cache_data_store.go index c296807e57..848ab96c7a 100644 --- a/cacheservice/pkg/manager/impl/cache_data_store.go +++ b/cacheservice/pkg/repositories/clients/cache_data_store.go @@ -1,4 +1,4 @@ -package impl +package clients import ( "context" @@ -13,50 +13,14 @@ import ( "google.golang.org/grpc/codes" "github.com/flyteorg/flyte/cacheservice/pkg/errors" - "github.com/flyteorg/flyte/cacheservice/pkg/manager/interfaces" + "github.com/flyteorg/flyte/cacheservice/pkg/repositories/interfaces" + "github.com/flyteorg/flyte/cacheservice/pkg/repositories/models" "github.com/flyteorg/flyte/cacheservice/pkg/runtime/configs" - "github.com/flyteorg/flyte/cacheservice/repositories/models" "github.com/flyteorg/flyte/flytestdlib/logger" ) var ( - _ interfaces.CacheDataStoreClient = &memClient{} -) - -type memClient struct { - cacheMap *map[string]*models.CachedOutput -} - -// Get returns the cached output for the given key. It returns an error if the key does not exist. -func (c *memClient) Get(ctx context.Context, key string) (*models.CachedOutput, error) { - cache := *c.cacheMap - if value, exists := cache[key]; exists { - return value, nil - } - - return nil, errors.NewNotFoundError("output", key) -} - -// Put will always set the value for the given key. It will overwrite the existing value if it exists. -func (c *memClient) Put(ctx context.Context, key string, cachedOutput *models.CachedOutput) error { - cache := *c.cacheMap - cache[key] = cachedOutput - return nil -} - -// Delete is an idempotent operation. It will not return an error if the key does not exist. -func (c *memClient) Delete(ctx context.Context, key string) error { - cache := *c.cacheMap - if _, exists := cache[key]; exists { - delete(cache, key) - return nil - } - - return errors.NewNotFoundError("output", key) -} - -var ( - _ interfaces.CacheDataStoreClient = &dynamoClient{} + _ interfaces.CachedOutputRepo = &dynamoClient{} ) type dynamoClient struct { @@ -129,8 +93,21 @@ func (c *dynamoClient) Delete(ctx context.Context, key string) error { return nil } +func NewDynamoCachedOutputRepo(ctx context.Context, serviceConfig configs.CacheServiceConfig) interfaces.CachedOutputRepo { + cfg, err := awsConfig.LoadDefaultConfig(context.TODO(), + awsConfig.WithRegion(serviceConfig.AwsRegion), + ) + if err != nil { + panic("unable to load AWS config to connect to Dynamo" + err.Error()) + } + return &dynamoClient{ + DynamoDbClient: dynamodb.NewFromConfig(cfg), + TableName: "cache", + } +} + var ( - _ interfaces.CacheDataStoreClient = &redisClient{} + _ interfaces.CachedOutputRepo = &redisClient{} ) type redisClient struct { @@ -186,41 +163,20 @@ func (r *redisClient) Delete(ctx context.Context, key string) error { return nil } -func NewCacheDataStore(ctx context.Context, serviceConfig configs.CacheServiceConfig) interfaces.CacheDataStoreClient { - clientType := serviceConfig.DataStoreType - - switch clientType { - case configs.Mem: - cacheMap := make(map[string]*models.CachedOutput) - return &memClient{cacheMap: &cacheMap} - case configs.DynamoDB: - cfg, err := awsConfig.LoadDefaultConfig(context.TODO(), - awsConfig.WithRegion(serviceConfig.AwsRegion), - ) - if err != nil { - panic("unable to load AWS config to connect to Dynamo" + err.Error()) - } - return &dynamoClient{ - DynamoDbClient: dynamodb.NewFromConfig(cfg), - TableName: "cache", - } - case configs.Redis: - rdb := redis.NewClient( - &redis.Options{ - Addr: serviceConfig.RedisAddress, - Username: serviceConfig.RedisUsername, - Password: serviceConfig.RedisPassword, - }) - - _, err := rdb.Ping(ctx).Result() - if err != nil { - panic("failed to connect to redis " + err.Error()) - } - - return &redisClient{ - RedisClient: rdb, - } - } - - panic("unsupported cache data store type") +func NewRedisCachedOutputRepo(ctx context.Context, serviceConfig configs.CacheServiceConfig) interfaces.CachedOutputRepo { + rdb := redis.NewClient( + &redis.Options{ + Addr: serviceConfig.RedisAddress, + Username: serviceConfig.RedisUsername, + Password: serviceConfig.RedisPassword, + }) + + _, err := rdb.Ping(ctx).Result() + if err != nil { + panic("failed to connect to redis " + err.Error()) + } + + return &redisClient{ + RedisClient: rdb, + } } diff --git a/cacheservice/pkg/repositories/clients/postgres/cached_output_repo.go b/cacheservice/pkg/repositories/clients/postgres/cached_output_repo.go new file mode 100644 index 0000000000..b6387e097e --- /dev/null +++ b/cacheservice/pkg/repositories/clients/postgres/cached_output_repo.go @@ -0,0 +1,95 @@ +package postgres + +import ( + "context" + "errors" + + "gorm.io/gorm" + "gorm.io/gorm/clause" + + cacheErr "github.com/flyteorg/flyte/cacheservice/pkg/errors" + errors2 "github.com/flyteorg/flyte/cacheservice/pkg/repositories/errors" + "github.com/flyteorg/flyte/cacheservice/pkg/repositories/interfaces" + "github.com/flyteorg/flyte/cacheservice/pkg/repositories/models" + "github.com/flyteorg/flyte/flytestdlib/promutils" +) + +var ( + _ interfaces.CachedOutputRepo = &postgresCachedOutputRepo{} +) + +type postgresCachedOutputRepo struct { + db *gorm.DB + errorTransformer errors2.ErrorTransformer + repoMetrics gormMetrics +} + +func (c postgresCachedOutputRepo) Get(ctx context.Context, key string) (*models.CachedOutput, error) { + timer := c.repoMetrics.GetDuration.Start(ctx) + defer timer.Stop() + + var cachedOutput models.CachedOutput + result := c.db.WithContext(ctx).Where(&models.CachedOutput{ + BaseModel: models.BaseModel{ + ID: key, + }, + }).Take(&cachedOutput) + if result.Error != nil { + if errors.Is(result.Error, gorm.ErrRecordNotFound) { + return nil, cacheErr.NewNotFoundError("output", key) + } + return nil, result.Error + } + return &cachedOutput, nil +} + +func (c postgresCachedOutputRepo) Put(ctx context.Context, key string, cachedOutput *models.CachedOutput) error { + timer := c.repoMetrics.PutDuration.Start(ctx) + defer timer.Stop() + + tx := c.db.WithContext(ctx).Begin() + + tx = tx.Clauses(clause.OnConflict{UpdateAll: true}).Create(&cachedOutput) + + if tx.Error != nil { + tx.Rollback() + return c.errorTransformer.ToCacheServiceError(tx.Error) + } + + tx = tx.Commit() + if tx.Error != nil { + return c.errorTransformer.ToCacheServiceError(tx.Error) + } + + return nil +} + +func (c postgresCachedOutputRepo) Delete(ctx context.Context, key string) error { + timer := c.repoMetrics.DeleteDuration.Start(ctx) + defer timer.Stop() + + var cachedOutput models.CachedOutput + + result := c.db.WithContext(ctx).Where(&models.CachedOutput{ + BaseModel: models.BaseModel{ + ID: key, + }, + }).Delete(&cachedOutput) + if result.Error != nil { + return c.errorTransformer.ToCacheServiceError(result.Error) + } + + if result.RowsAffected == 0 { + return cacheErr.NewNotFoundError("output", key) + } + + return nil +} + +func NewPostgresOutputRepo(db *gorm.DB, scope promutils.Scope) interfaces.CachedOutputRepo { + return &postgresCachedOutputRepo{ + db: db, + errorTransformer: errors2.NewPostgresErrorTransformer(), + repoMetrics: newPostgresRepoMetrics(scope.NewSubScope("cached_output")), + } +} diff --git a/cacheservice/pkg/repositories/clients/postgres/cached_output_repo_test.go b/cacheservice/pkg/repositories/clients/postgres/cached_output_repo_test.go new file mode 100644 index 0000000000..ab5b962c9c --- /dev/null +++ b/cacheservice/pkg/repositories/clients/postgres/cached_output_repo_test.go @@ -0,0 +1,73 @@ +package postgres + +import ( + "context" + "testing" + + mocket "github.com/Selvatico/go-mocket" + "github.com/stretchr/testify/assert" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "gorm.io/gorm" + + "github.com/flyteorg/flyte/flytestdlib/contextutils" + "github.com/flyteorg/flyte/flytestdlib/promutils" + "github.com/flyteorg/flyte/flytestdlib/promutils/labeled" +) + +func init() { + labeled.SetMetricKeys(contextutils.AppNameKey) +} + +var sampleKey = "sampleKey" + +func TestGetOutputNotFound(t *testing.T) { + ctx := context.Background() + + cachedOutputRepo := NewPostgresOutputRepo(GetDbForTest(t), promutils.NewTestScope()) + result, err := cachedOutputRepo.Get(ctx, sampleKey) + + assert.Error(t, err) + assert.Nil(t, result) + assert.EqualValues(t, codes.NotFound, status.Code(err)) + +} + +func TestGetOutputError(t *testing.T) { + ctx := context.Background() + + GlobalMock := mocket.Catcher.Reset() + GlobalMock.Logging = true + + GlobalMock.NewMock().WithQuery(`SELECT * FROM "cached_outputs" WHERE "cached_outputs"."id" = $1 LIMIT 1`).WithReply([]map[string]interface{}{ + { + "id": "sampleKey", + }, + }).WithError(gorm.ErrInvalidDB) + + cachedOutputRepo := NewPostgresOutputRepo(GetDbForTest(t), promutils.NewTestScope()) + result, err := cachedOutputRepo.Get(ctx, sampleKey) + + assert.Error(t, err) + assert.Nil(t, result) + assert.NotEqualf(t, codes.NotFound, status.Code(err), "Expected error other than NotFound") + +} + +func TestGetOutput(t *testing.T) { + ctx := context.Background() + + GlobalMock := mocket.Catcher.Reset() + GlobalMock.Logging = true + + GlobalMock.NewMock().WithQuery(`SELECT * FROM "cached_outputs" WHERE "cached_outputs"."id" = $1 LIMIT 1`).WithReply([]map[string]interface{}{ + { + "id": "sampleKey", + }, + }) + cachedOutputRepo := NewPostgresOutputRepo(GetDbForTest(t), promutils.NewTestScope()) + result, err := cachedOutputRepo.Get(ctx, sampleKey) + + assert.NoError(t, err) + assert.NotNil(t, result) +} diff --git a/cacheservice/pkg/repositories/clients/postgres/handle.go b/cacheservice/pkg/repositories/clients/postgres/handle.go new file mode 100644 index 0000000000..c18a2f02a5 --- /dev/null +++ b/cacheservice/pkg/repositories/clients/postgres/handle.go @@ -0,0 +1,39 @@ +package postgres + +import ( + "context" + + "gorm.io/gorm" + + "github.com/flyteorg/flyte/cacheservice/pkg/repositories/models" + "github.com/flyteorg/flyte/flytestdlib/database" + "github.com/flyteorg/flyte/flytestdlib/logger" +) + +type DBHandle struct { + DB *gorm.DB +} + +func NewDBHandle(ctx context.Context, dbConfigValues *database.DbConfig) (*DBHandle, error) { + logConfig := logger.GetConfig() + db, err := database.GetDB(ctx, dbConfigValues, logConfig) + if err != nil { + return nil, err + } + + return &DBHandle{ + DB: db, + }, nil +} + +func (h *DBHandle) Migrate(ctx context.Context) error { + if err := h.DB.AutoMigrate(&models.CachedOutput{}); err != nil { + return err + } + + if err := h.DB.Debug().AutoMigrate(&models.CacheReservation{}); err != nil { + return err + } + + return nil +} diff --git a/cacheservice/pkg/repositories/clients/postgres/metrics.go b/cacheservice/pkg/repositories/clients/postgres/metrics.go new file mode 100644 index 0000000000..d8378dfddf --- /dev/null +++ b/cacheservice/pkg/repositories/clients/postgres/metrics.go @@ -0,0 +1,39 @@ +package postgres + +import ( + "time" + + "github.com/flyteorg/flyte/flytestdlib/promutils" + "github.com/flyteorg/flyte/flytestdlib/promutils/labeled" +) + +type gormMetrics struct { + Scope promutils.Scope + PutDuration labeled.StopWatch + GetDuration labeled.StopWatch + DeleteDuration labeled.StopWatch + CreateReservationDuration labeled.StopWatch + UpdateReservationDuration labeled.StopWatch + DeleteReservationDuration labeled.StopWatch + GetReservationDuration labeled.StopWatch +} + +func newPostgresRepoMetrics(scope promutils.Scope) gormMetrics { + return gormMetrics{ + Scope: scope, + PutDuration: labeled.NewStopWatch( + "put", "time taken to create/update a new entry", time.Millisecond, scope), + GetDuration: labeled.NewStopWatch( + "get", "time taken to get an entry", time.Millisecond, scope), + DeleteDuration: labeled.NewStopWatch( + "delete", "time taken to delete an individual entry", time.Millisecond, scope), + CreateReservationDuration: labeled.NewStopWatch( + "createReservation", "time taken to create a new entry", time.Millisecond, scope), + UpdateReservationDuration: labeled.NewStopWatch( + "updateReservation", "time taken to update an entry", time.Millisecond, scope), + DeleteReservationDuration: labeled.NewStopWatch( + "deleteReservation", "time taken to delete an individual entry", time.Millisecond, scope), + GetReservationDuration: labeled.NewStopWatch( + "getReservation", "time taken to get an entry", time.Millisecond, scope), + } +} diff --git a/cacheservice/pkg/repositories/clients/postgres/reservation_repo.go b/cacheservice/pkg/repositories/clients/postgres/reservation_repo.go new file mode 100644 index 0000000000..d898d7c54e --- /dev/null +++ b/cacheservice/pkg/repositories/clients/postgres/reservation_repo.go @@ -0,0 +1,108 @@ +package postgres + +import ( + "context" + "time" + + "google.golang.org/grpc/codes" + "gorm.io/gorm" + "gorm.io/gorm/clause" + + cacheErr "github.com/flyteorg/flyte/cacheservice/pkg/errors" + errors2 "github.com/flyteorg/flyte/cacheservice/pkg/repositories/errors" + "github.com/flyteorg/flyte/cacheservice/pkg/repositories/interfaces" + "github.com/flyteorg/flyte/cacheservice/pkg/repositories/models" + "github.com/flyteorg/flyte/flytestdlib/promutils" +) + +var ( + _ interfaces.ReservationRepo = &reservationRepo{} +) + +type reservationRepo struct { + db *gorm.DB + errorTransformer errors2.ErrorTransformer + repoMetrics gormMetrics +} + +func (r reservationRepo) Create(ctx context.Context, reservation *models.CacheReservation, now time.Time) error { + timer := r.repoMetrics.CreateReservationDuration.Start(ctx) + defer timer.Stop() + + result := r.db.WithContext(ctx).Clauses(clause.OnConflict{DoNothing: true}).Create(&reservation) + if result.Error != nil { + return r.errorTransformer.ToCacheServiceError(result.Error) + } + + if result.RowsAffected == 0 { + return cacheErr.NewCacheServiceErrorf(codes.AlreadyExists, "reservation with key %s already exists", reservation.Key) + } + + return nil +} + +func (r reservationRepo) Update(ctx context.Context, reservation *models.CacheReservation, now time.Time) error { + timer := r.repoMetrics.UpdateReservationDuration.Start(ctx) + defer timer.Stop() + + result := r.db.WithContext(ctx).Model(&models.CacheReservation{ + Key: reservation.Key, + }).Where("expires_at<=? OR owner_id=?", now, reservation.OwnerID).Updates(reservation) + if result.Error != nil { + return r.errorTransformer.ToCacheServiceError(result.Error) + } + + if result.RowsAffected == 0 { + return cacheErr.NewCacheServiceErrorf(codes.AlreadyExists, "reservation with key %s already exists", reservation.Key) + } + + return nil +} + +func (r reservationRepo) Get(ctx context.Context, key string) (*models.CacheReservation, error) { + timer := r.repoMetrics.GetDuration.Start(ctx) + defer timer.Stop() + + var reservation models.CacheReservation + result := r.db.WithContext(ctx).Where(&models.CacheReservation{ + Key: key, + }).Take(&reservation) + + if result.Error != nil { + return &reservation, r.errorTransformer.ToCacheServiceError(result.Error) + } + + if result.RowsAffected == 0 { + return nil, cacheErr.NewNotFoundError("reservation", key) + } + + return &reservation, nil +} + +func (r reservationRepo) Delete(ctx context.Context, key string, ownerID string) error { + timer := r.repoMetrics.DeleteReservationDuration.Start(ctx) + defer timer.Stop() + + var reservation models.CacheReservation + result := r.db.WithContext(ctx).Where(&models.CacheReservation{ + Key: key, + OwnerID: ownerID, + }).Delete(&reservation) + if result.Error != nil { + return r.errorTransformer.ToCacheServiceError(result.Error) + } + + if result.RowsAffected == 0 { + return cacheErr.NewNotFoundError("reservation", key) + } + + return nil +} + +func NewReservationRepo(db *gorm.DB, scope promutils.Scope) interfaces.ReservationRepo { + return &reservationRepo{ + db: db, + errorTransformer: errors2.NewPostgresErrorTransformer(), + repoMetrics: newPostgresRepoMetrics(scope.NewSubScope("reservation")), + } +} diff --git a/cacheservice/pkg/repositories/clients/postgres/reservation_repo_test.go b/cacheservice/pkg/repositories/clients/postgres/reservation_repo_test.go new file mode 100644 index 0000000000..bf560bea28 --- /dev/null +++ b/cacheservice/pkg/repositories/clients/postgres/reservation_repo_test.go @@ -0,0 +1 @@ +package postgres diff --git a/cacheservice/pkg/repositories/clients/postgres/test_utils.go b/cacheservice/pkg/repositories/clients/postgres/test_utils.go new file mode 100644 index 0000000000..9ff6e50ef5 --- /dev/null +++ b/cacheservice/pkg/repositories/clients/postgres/test_utils.go @@ -0,0 +1,18 @@ +package postgres + +import ( + "testing" + + mocket "github.com/Selvatico/go-mocket" + "gorm.io/driver/postgres" + "gorm.io/gorm" +) + +func GetDbForTest(t *testing.T) *gorm.DB { + mocket.Catcher.Register() + db, err := gorm.Open(postgres.New(postgres.Config{DriverName: mocket.DriverName})) + if err != nil { + t.Fatalf("Failed to open mock db with err %v", err) + } + return db +} diff --git a/cacheservice/pkg/manager/impl/reservation_store.go b/cacheservice/pkg/repositories/clients/reservation_store.go similarity index 52% rename from cacheservice/pkg/manager/impl/reservation_store.go rename to cacheservice/pkg/repositories/clients/reservation_store.go index 57c6eadd99..3ad1003f43 100644 --- a/cacheservice/pkg/manager/impl/reservation_store.go +++ b/cacheservice/pkg/repositories/clients/reservation_store.go @@ -1,4 +1,4 @@ -package impl +package clients import ( "context" @@ -9,51 +9,13 @@ import ( "google.golang.org/grpc/codes" "github.com/flyteorg/flyte/cacheservice/pkg/errors" - "github.com/flyteorg/flyte/cacheservice/pkg/manager/interfaces" + "github.com/flyteorg/flyte/cacheservice/pkg/repositories/interfaces" + "github.com/flyteorg/flyte/cacheservice/pkg/repositories/models" "github.com/flyteorg/flyte/cacheservice/pkg/runtime/configs" - "github.com/flyteorg/flyte/cacheservice/repositories/models" ) var ( - _ interfaces.ReservationDataStoreClient = &reservationMemClient{} -) - -type reservationMemClient struct { - keyValStore *map[string]*models.Reservation -} - -func (r reservationMemClient) Create(ctx context.Context, reservation *models.Reservation, now time.Time) error { - reservationStore := *r.keyValStore - if _, exists := reservationStore[reservation.Key]; exists { - return errors.NewCacheServiceErrorf(codes.AlreadyExists, "reservation with key %s already exists", reservation.Key) - } - reservationStore[reservation.Key] = reservation - return nil -} - -func (r reservationMemClient) Update(ctx context.Context, reservation *models.Reservation, now time.Time) error { - reservationStore := *r.keyValStore - reservationStore[reservation.Key] = reservation - return nil -} - -func (r reservationMemClient) Get(ctx context.Context, key string) (*models.Reservation, error) { - reservationStore := *r.keyValStore - if value, exists := reservationStore[key]; exists { - return value, nil - } - - return nil, errors.NewNotFoundError("reservation", key) -} - -func (r reservationMemClient) Delete(ctx context.Context, key string, ownerID string) error { - reservationStore := *r.keyValStore - delete(reservationStore, key) - return nil -} - -var ( - _ interfaces.ReservationDataStoreClient = &reservationRedisClient{} + _ interfaces.ReservationRepo = &reservationRedisClient{} ) type reservationRedisClient struct { @@ -61,7 +23,7 @@ type reservationRedisClient struct { } // Create a new reservation in the data store. If the reservation already exists, return an error. -func (r *reservationRedisClient) Create(ctx context.Context, reservation *models.Reservation, now time.Time) error { +func (r *reservationRedisClient) Create(ctx context.Context, reservation *models.CacheReservation, now time.Time) error { ttl := reservation.ExpiresAt.Sub(now) marshaledReservation, err := json.Marshal(reservation) @@ -79,7 +41,7 @@ func (r *reservationRedisClient) Create(ctx context.Context, reservation *models return errors.NewCacheServiceErrorf(codes.Internal, "Failed to confirm setting of reservation with key %s", reservation.Key) } if !wasSet { - return errors.NewCacheServiceErrorf(codes.AlreadyExists, "Reservation with key %s already exists", reservation.Key) + return errors.NewCacheServiceErrorf(codes.AlreadyExists, "CacheReservation with key %s already exists", reservation.Key) } return nil @@ -87,7 +49,7 @@ func (r *reservationRedisClient) Create(ctx context.Context, reservation *models // Update acts as an upsert operation. Since we already check in cache_manager for collision handling, we reduce the // number of calls to Redis by upsert-ing as opposed to inserting -func (r *reservationRedisClient) Update(ctx context.Context, reservation *models.Reservation, now time.Time) error { +func (r *reservationRedisClient) Update(ctx context.Context, reservation *models.CacheReservation, now time.Time) error { ttl := reservation.ExpiresAt.Sub(now) marshaledReservation, err := json.Marshal(reservation) @@ -104,7 +66,7 @@ func (r *reservationRedisClient) Update(ctx context.Context, reservation *models } // Get a reservation from the data store. If the reservation does not exist, return an error. -func (r *reservationRedisClient) Get(ctx context.Context, key string) (*models.Reservation, error) { +func (r *reservationRedisClient) Get(ctx context.Context, key string) (*models.CacheReservation, error) { val, err := r.RedisClient.Get(ctx, key).Result() if err == redis.Nil { return nil, errors.NewNotFoundError("reservation", key) @@ -112,7 +74,7 @@ func (r *reservationRedisClient) Get(ctx context.Context, key string) (*models.R return nil, errors.NewCacheServiceErrorf(codes.Internal, "Failed to get reservation with key %s", key) } - var reservation models.Reservation + var reservation models.CacheReservation err = json.Unmarshal([]byte(val), &reservation) if err != nil { return nil, errors.NewCacheServiceErrorf(codes.Internal, "Failed to unmarshal reservation with error %v", err) @@ -130,32 +92,20 @@ func (r *reservationRedisClient) Delete(ctx context.Context, key string, ownerID return nil } -func NewReservationDataStore(ctx context.Context, serviceConfig configs.CacheServiceConfig) interfaces.ReservationDataStoreClient { - clientType := serviceConfig.ReservationDataStoreType - - switch clientType { - case configs.Mem: - keyValStore := make(map[string]*models.Reservation) - return &reservationMemClient{keyValStore: &keyValStore} - case configs.DynamoDB: - panic("dynamodb not supported for reservation data store") - case configs.Redis: - rdb := redis.NewClient( - &redis.Options{ - Addr: serviceConfig.RedisAddress, - Username: serviceConfig.RedisUsername, - Password: serviceConfig.RedisPassword, - }) - - _, err := rdb.Ping(ctx).Result() - if err != nil { - panic("failed to connect to redis " + err.Error()) - } - - return &reservationRedisClient{ - RedisClient: rdb, - } +func NewRedisReservationRepo(ctx context.Context, serviceConfig configs.CacheServiceConfig) interfaces.ReservationRepo { + rdb := redis.NewClient( + &redis.Options{ + Addr: serviceConfig.RedisAddress, + Username: serviceConfig.RedisUsername, + Password: serviceConfig.RedisPassword, + }) + + _, err := rdb.Ping(ctx).Result() + if err != nil { + panic("failed to connect to redis " + err.Error()) } - panic("unsupported reservation data store type") + return &reservationRedisClient{ + RedisClient: rdb, + } } diff --git a/cacheservice/pkg/repositories/errors/errors.go b/cacheservice/pkg/repositories/errors/errors.go new file mode 100644 index 0000000000..359fc6a88e --- /dev/null +++ b/cacheservice/pkg/repositories/errors/errors.go @@ -0,0 +1,5 @@ +package errors + +type ErrorTransformer interface { + ToCacheServiceError(err error) error +} diff --git a/cacheservice/pkg/repositories/errors/postgres_transformer.go b/cacheservice/pkg/repositories/errors/postgres_transformer.go new file mode 100644 index 0000000000..6b11844b0a --- /dev/null +++ b/cacheservice/pkg/repositories/errors/postgres_transformer.go @@ -0,0 +1,71 @@ +package errors + +import ( + "errors" + "fmt" + "reflect" + + "github.com/jackc/pgconn" + "google.golang.org/grpc/codes" + "gorm.io/gorm" + + cacheErr "github.com/flyteorg/flyte/cacheservice/pkg/errors" + "github.com/flyteorg/flyte/flytestdlib/database" + "github.com/flyteorg/flyte/flytestdlib/logger" +) + +// Postgres error codes +const ( + uniqueConstraintViolationCode = "23505" + undefinedTable = "42P01" +) + +type postgresErrorTransformer struct { +} + +const ( + unexpectedType = "unexpected error type for: %v" + uniqueConstraintViolation = "value with matching already exists (%s)" + defaultPgError = "failed database operation with code [%s] and msg [%s]" + unsupportedTableOperation = "cannot query with specified table attributes: %s" +) + +func (p *postgresErrorTransformer) fromGormError(err error) error { + switch err.Error() { + case gorm.ErrRecordNotFound.Error(): + return cacheErr.NewCacheServiceErrorf(codes.NotFound, "entry not found") + default: + return cacheErr.NewCacheServiceErrorf(codes.Internal, unexpectedType, err) + } +} + +func (p *postgresErrorTransformer) ToCacheServiceError(err error) error { + // First try the stdlib error handling + if database.IsPgErrorWithCode(err, uniqueConstraintViolationCode) { + return cacheErr.NewCacheServiceErrorf(codes.AlreadyExists, uniqueConstraintViolation, err.Error()) + } + + if unwrappedErr := errors.Unwrap(err); unwrappedErr != nil { + err = unwrappedErr + } + + pqError, ok := err.(*pgconn.PgError) + if !ok { + logger.InfofNoCtx("Unable to cast to pgconn.PgError. Error type: [%v]", + reflect.TypeOf(err)) + return p.fromGormError(err) + } + + switch pqError.Code { + case uniqueConstraintViolationCode: + return cacheErr.NewCacheServiceErrorf(codes.AlreadyExists, uniqueConstraintViolation, pqError.Message) + case undefinedTable: + return cacheErr.NewCacheServiceErrorf(codes.InvalidArgument, unsupportedTableOperation, pqError.Message) + default: + return cacheErr.NewCacheServiceErrorf(codes.Unknown, fmt.Sprintf(defaultPgError, pqError.Code, pqError.Message)) + } +} + +func NewPostgresErrorTransformer() ErrorTransformer { + return &postgresErrorTransformer{} +} diff --git a/cacheservice/pkg/repositories/initialize.go b/cacheservice/pkg/repositories/initialize.go new file mode 100644 index 0000000000..b2e56f15e1 --- /dev/null +++ b/cacheservice/pkg/repositories/initialize.go @@ -0,0 +1,35 @@ +package repositories + +import ( + "context" + + "github.com/flyteorg/flyte/cacheservice/pkg/repositories/clients/postgres" + "github.com/flyteorg/flyte/cacheservice/pkg/runtime" + "github.com/flyteorg/flyte/flytestdlib/logger" +) + +// Migrate This command will run all the migrations for the database +func Migrate(ctx context.Context) error { + configProvider := runtime.NewConfigurationProvider() + + clientType := configProvider.ApplicationConfiguration().GetCacheServiceConfig().OutputDataStoreType + + switch clientType { + case "postgres": + dbConfigValues := configProvider.ApplicationConfiguration().GetDbConfig() + dbHandle, err := postgres.NewDBHandle(ctx, dbConfigValues) + if err != nil { + logger.Errorf(ctx, "Failed to get DB connection. err: %v", err) + panic(err) + } + + logger.Infof(ctx, "Created DB connection.") + + if err := dbHandle.Migrate(ctx); err != nil { + logger.Errorf(ctx, "Failed to migrate. err: %v", err) + panic(err) + } + logger.Infof(ctx, "Ran DB migration successfully.") + } + return nil +} diff --git a/cacheservice/pkg/repositories/interfaces/cache.go b/cacheservice/pkg/repositories/interfaces/cache.go new file mode 100644 index 0000000000..1f2f74bc91 --- /dev/null +++ b/cacheservice/pkg/repositories/interfaces/cache.go @@ -0,0 +1,28 @@ +package interfaces + +import ( + "context" + "time" + + "github.com/flyteorg/flyte/cacheservice/pkg/repositories/models" +) + +//go:generate mockery -all -output=../mocks -case=underscore + +type CachedOutputRepo interface { + Get(ctx context.Context, key string) (*models.CachedOutput, error) + Put(ctx context.Context, key string, cachedOutput *models.CachedOutput) error + Delete(ctx context.Context, key string) error +} + +type ReservationRepo interface { + Create(ctx context.Context, reservation *models.CacheReservation, now time.Time) error + Update(ctx context.Context, reservation *models.CacheReservation, now time.Time) error + Get(ctx context.Context, key string) (*models.CacheReservation, error) + Delete(ctx context.Context, key string, ownerID string) error +} + +type RepositoryInterface interface { + CachedOutputRepo() CachedOutputRepo + ReservationRepo() ReservationRepo +} diff --git a/cacheservice/pkg/repositories/mocks/cached_output_repo.go b/cacheservice/pkg/repositories/mocks/cached_output_repo.go new file mode 100644 index 0000000000..9543ce8a46 --- /dev/null +++ b/cacheservice/pkg/repositories/mocks/cached_output_repo.go @@ -0,0 +1,121 @@ +// Code generated by mockery v1.0.1. DO NOT EDIT. + +package mocks + +import ( + context "context" + + mock "github.com/stretchr/testify/mock" + + models "github.com/flyteorg/flyte/cacheservice/pkg/repositories/models" +) + +// CachedOutputRepo is an autogenerated mock type for the CachedOutputRepo type +type CachedOutputRepo struct { + mock.Mock +} + +type CachedOutputRepo_Delete struct { + *mock.Call +} + +func (_m CachedOutputRepo_Delete) Return(_a0 error) *CachedOutputRepo_Delete { + return &CachedOutputRepo_Delete{Call: _m.Call.Return(_a0)} +} + +func (_m *CachedOutputRepo) OnDelete(ctx context.Context, key string) *CachedOutputRepo_Delete { + c_call := _m.On("Delete", ctx, key) + return &CachedOutputRepo_Delete{Call: c_call} +} + +func (_m *CachedOutputRepo) OnDeleteMatch(matchers ...interface{}) *CachedOutputRepo_Delete { + c_call := _m.On("Delete", matchers...) + return &CachedOutputRepo_Delete{Call: c_call} +} + +// Delete provides a mock function with given fields: ctx, key +func (_m *CachedOutputRepo) Delete(ctx context.Context, key string) error { + ret := _m.Called(ctx, key) + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, string) error); ok { + r0 = rf(ctx, key) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +type CachedOutputRepo_Get struct { + *mock.Call +} + +func (_m CachedOutputRepo_Get) Return(_a0 *models.CachedOutput, _a1 error) *CachedOutputRepo_Get { + return &CachedOutputRepo_Get{Call: _m.Call.Return(_a0, _a1)} +} + +func (_m *CachedOutputRepo) OnGet(ctx context.Context, key string) *CachedOutputRepo_Get { + c_call := _m.On("Get", ctx, key) + return &CachedOutputRepo_Get{Call: c_call} +} + +func (_m *CachedOutputRepo) OnGetMatch(matchers ...interface{}) *CachedOutputRepo_Get { + c_call := _m.On("Get", matchers...) + return &CachedOutputRepo_Get{Call: c_call} +} + +// Get provides a mock function with given fields: ctx, key +func (_m *CachedOutputRepo) Get(ctx context.Context, key string) (*models.CachedOutput, error) { + ret := _m.Called(ctx, key) + + var r0 *models.CachedOutput + if rf, ok := ret.Get(0).(func(context.Context, string) *models.CachedOutput); ok { + r0 = rf(ctx, key) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*models.CachedOutput) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, string) error); ok { + r1 = rf(ctx, key) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +type CachedOutputRepo_Put struct { + *mock.Call +} + +func (_m CachedOutputRepo_Put) Return(_a0 error) *CachedOutputRepo_Put { + return &CachedOutputRepo_Put{Call: _m.Call.Return(_a0)} +} + +func (_m *CachedOutputRepo) OnPut(ctx context.Context, key string, cachedOutput *models.CachedOutput) *CachedOutputRepo_Put { + c_call := _m.On("Put", ctx, key, cachedOutput) + return &CachedOutputRepo_Put{Call: c_call} +} + +func (_m *CachedOutputRepo) OnPutMatch(matchers ...interface{}) *CachedOutputRepo_Put { + c_call := _m.On("Put", matchers...) + return &CachedOutputRepo_Put{Call: c_call} +} + +// Put provides a mock function with given fields: ctx, key, cachedOutput +func (_m *CachedOutputRepo) Put(ctx context.Context, key string, cachedOutput *models.CachedOutput) error { + ret := _m.Called(ctx, key, cachedOutput) + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, string, *models.CachedOutput) error); ok { + r0 = rf(ctx, key, cachedOutput) + } else { + r0 = ret.Error(0) + } + + return r0 +} diff --git a/cacheservice/pkg/repositories/mocks/repository_interface.go b/cacheservice/pkg/repositories/mocks/repository_interface.go new file mode 100644 index 0000000000..087f998e8e --- /dev/null +++ b/cacheservice/pkg/repositories/mocks/repository_interface.go @@ -0,0 +1,81 @@ +// Code generated by mockery v1.0.1. DO NOT EDIT. + +package mocks + +import ( + interfaces "github.com/flyteorg/flyte/cacheservice/pkg/repositories/interfaces" + mock "github.com/stretchr/testify/mock" +) + +// RepositoryInterface is an autogenerated mock type for the RepositoryInterface type +type RepositoryInterface struct { + mock.Mock +} + +type RepositoryInterface_CachedOutputRepo struct { + *mock.Call +} + +func (_m RepositoryInterface_CachedOutputRepo) Return(_a0 interfaces.CachedOutputRepo) *RepositoryInterface_CachedOutputRepo { + return &RepositoryInterface_CachedOutputRepo{Call: _m.Call.Return(_a0)} +} + +func (_m *RepositoryInterface) OnCachedOutputRepo() *RepositoryInterface_CachedOutputRepo { + c_call := _m.On("CachedOutputRepo") + return &RepositoryInterface_CachedOutputRepo{Call: c_call} +} + +func (_m *RepositoryInterface) OnCachedOutputRepoMatch(matchers ...interface{}) *RepositoryInterface_CachedOutputRepo { + c_call := _m.On("CachedOutputRepo", matchers...) + return &RepositoryInterface_CachedOutputRepo{Call: c_call} +} + +// CachedOutputRepo provides a mock function with given fields: +func (_m *RepositoryInterface) CachedOutputRepo() interfaces.CachedOutputRepo { + ret := _m.Called() + + var r0 interfaces.CachedOutputRepo + if rf, ok := ret.Get(0).(func() interfaces.CachedOutputRepo); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(interfaces.CachedOutputRepo) + } + } + + return r0 +} + +type RepositoryInterface_ReservationRepo struct { + *mock.Call +} + +func (_m RepositoryInterface_ReservationRepo) Return(_a0 interfaces.ReservationRepo) *RepositoryInterface_ReservationRepo { + return &RepositoryInterface_ReservationRepo{Call: _m.Call.Return(_a0)} +} + +func (_m *RepositoryInterface) OnReservationRepo() *RepositoryInterface_ReservationRepo { + c_call := _m.On("ReservationRepo") + return &RepositoryInterface_ReservationRepo{Call: c_call} +} + +func (_m *RepositoryInterface) OnReservationRepoMatch(matchers ...interface{}) *RepositoryInterface_ReservationRepo { + c_call := _m.On("ReservationRepo", matchers...) + return &RepositoryInterface_ReservationRepo{Call: c_call} +} + +// ReservationRepo provides a mock function with given fields: +func (_m *RepositoryInterface) ReservationRepo() interfaces.ReservationRepo { + ret := _m.Called() + + var r0 interfaces.ReservationRepo + if rf, ok := ret.Get(0).(func() interfaces.ReservationRepo); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(interfaces.ReservationRepo) + } + } + + return r0 +} diff --git a/cacheservice/pkg/repositories/mocks/reservation_repo.go b/cacheservice/pkg/repositories/mocks/reservation_repo.go new file mode 100644 index 0000000000..747cce6f8d --- /dev/null +++ b/cacheservice/pkg/repositories/mocks/reservation_repo.go @@ -0,0 +1,155 @@ +// Code generated by mockery v1.0.1. DO NOT EDIT. + +package mocks + +import ( + context "context" + + mock "github.com/stretchr/testify/mock" + + models "github.com/flyteorg/flyte/cacheservice/pkg/repositories/models" + + time "time" +) + +// ReservationRepo is an autogenerated mock type for the ReservationRepo type +type ReservationRepo struct { + mock.Mock +} + +type ReservationRepo_Create struct { + *mock.Call +} + +func (_m ReservationRepo_Create) Return(_a0 error) *ReservationRepo_Create { + return &ReservationRepo_Create{Call: _m.Call.Return(_a0)} +} + +func (_m *ReservationRepo) OnCreate(ctx context.Context, reservation *models.CacheReservation, now time.Time) *ReservationRepo_Create { + c_call := _m.On("Create", ctx, reservation, now) + return &ReservationRepo_Create{Call: c_call} +} + +func (_m *ReservationRepo) OnCreateMatch(matchers ...interface{}) *ReservationRepo_Create { + c_call := _m.On("Create", matchers...) + return &ReservationRepo_Create{Call: c_call} +} + +// Create provides a mock function with given fields: ctx, reservation, now +func (_m *ReservationRepo) Create(ctx context.Context, reservation *models.CacheReservation, now time.Time) error { + ret := _m.Called(ctx, reservation, now) + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, *models.CacheReservation, time.Time) error); ok { + r0 = rf(ctx, reservation, now) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +type ReservationRepo_Delete struct { + *mock.Call +} + +func (_m ReservationRepo_Delete) Return(_a0 error) *ReservationRepo_Delete { + return &ReservationRepo_Delete{Call: _m.Call.Return(_a0)} +} + +func (_m *ReservationRepo) OnDelete(ctx context.Context, key string, ownerID string) *ReservationRepo_Delete { + c_call := _m.On("Delete", ctx, key, ownerID) + return &ReservationRepo_Delete{Call: c_call} +} + +func (_m *ReservationRepo) OnDeleteMatch(matchers ...interface{}) *ReservationRepo_Delete { + c_call := _m.On("Delete", matchers...) + return &ReservationRepo_Delete{Call: c_call} +} + +// Delete provides a mock function with given fields: ctx, key, ownerID +func (_m *ReservationRepo) Delete(ctx context.Context, key string, ownerID string) error { + ret := _m.Called(ctx, key, ownerID) + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, string, string) error); ok { + r0 = rf(ctx, key, ownerID) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +type ReservationRepo_Get struct { + *mock.Call +} + +func (_m ReservationRepo_Get) Return(_a0 *models.CacheReservation, _a1 error) *ReservationRepo_Get { + return &ReservationRepo_Get{Call: _m.Call.Return(_a0, _a1)} +} + +func (_m *ReservationRepo) OnGet(ctx context.Context, key string) *ReservationRepo_Get { + c_call := _m.On("Get", ctx, key) + return &ReservationRepo_Get{Call: c_call} +} + +func (_m *ReservationRepo) OnGetMatch(matchers ...interface{}) *ReservationRepo_Get { + c_call := _m.On("Get", matchers...) + return &ReservationRepo_Get{Call: c_call} +} + +// Get provides a mock function with given fields: ctx, key +func (_m *ReservationRepo) Get(ctx context.Context, key string) (*models.CacheReservation, error) { + ret := _m.Called(ctx, key) + + var r0 *models.CacheReservation + if rf, ok := ret.Get(0).(func(context.Context, string) *models.CacheReservation); ok { + r0 = rf(ctx, key) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*models.CacheReservation) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, string) error); ok { + r1 = rf(ctx, key) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +type ReservationRepo_Update struct { + *mock.Call +} + +func (_m ReservationRepo_Update) Return(_a0 error) *ReservationRepo_Update { + return &ReservationRepo_Update{Call: _m.Call.Return(_a0)} +} + +func (_m *ReservationRepo) OnUpdate(ctx context.Context, reservation *models.CacheReservation, now time.Time) *ReservationRepo_Update { + c_call := _m.On("Update", ctx, reservation, now) + return &ReservationRepo_Update{Call: c_call} +} + +func (_m *ReservationRepo) OnUpdateMatch(matchers ...interface{}) *ReservationRepo_Update { + c_call := _m.On("Update", matchers...) + return &ReservationRepo_Update{Call: c_call} +} + +// Update provides a mock function with given fields: ctx, reservation, now +func (_m *ReservationRepo) Update(ctx context.Context, reservation *models.CacheReservation, now time.Time) error { + ret := _m.Called(ctx, reservation, now) + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, *models.CacheReservation, time.Time) error); ok { + r0 = rf(ctx, reservation, now) + } else { + r0 = ret.Error(0) + } + + return r0 +} diff --git a/cacheservice/pkg/repositories/models/cached_output.go b/cacheservice/pkg/repositories/models/cached_output.go new file mode 100644 index 0000000000..5d89aa5b54 --- /dev/null +++ b/cacheservice/pkg/repositories/models/cached_output.go @@ -0,0 +1,31 @@ +package models + +import ( + "time" + + "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/core" +) + +type BaseModel struct { + ID string `gorm:"primary_key" dynamodbav:"id"` + CreatedAt time.Time + UpdatedAt time.Time + DeletedAt *time.Time `gorm:"index"` +} + +type Identifier struct { + ResourceType core.ResourceType `dynamodbav:"resourceType"` + Project string `dynamodbav:"project"` + Domain string `dynamodbav:"domain"` + Name string `dynamodbav:"name"` + Version string `dynamodbav:"version"` + Org string `dynamodbav:"org"` +} + +type CachedOutput struct { + BaseModel + OutputURI string `dynamodbav:"outputUri"` + OutputLiteral []byte `dynamodbav:"outputLiteral"` + SerializedMetadata []byte `dynamodbav:"keyMap"` + Identifier +} diff --git a/cacheservice/repositories/models/reservation.go b/cacheservice/pkg/repositories/models/reservation.go similarity index 52% rename from cacheservice/repositories/models/reservation.go rename to cacheservice/pkg/repositories/models/reservation.go index c713b550fd..bd9099f63f 100644 --- a/cacheservice/repositories/models/reservation.go +++ b/cacheservice/pkg/repositories/models/reservation.go @@ -4,8 +4,8 @@ import ( "time" ) -type Reservation struct { - Key string +type CacheReservation struct { + Key string `gorm:"primary_key"` OwnerID string ExpiresAt time.Time } diff --git a/cacheservice/pkg/repositories/repo_factory.go b/cacheservice/pkg/repositories/repo_factory.go new file mode 100644 index 0000000000..d938bb8cfb --- /dev/null +++ b/cacheservice/pkg/repositories/repo_factory.go @@ -0,0 +1,149 @@ +package repositories + +import ( + "context" + "time" + + "google.golang.org/grpc/codes" + "gorm.io/gorm" + + "github.com/flyteorg/flyte/cacheservice/pkg/errors" + "github.com/flyteorg/flyte/cacheservice/pkg/repositories/clients/postgres" + "github.com/flyteorg/flyte/cacheservice/pkg/repositories/interfaces" + "github.com/flyteorg/flyte/cacheservice/pkg/repositories/models" + "github.com/flyteorg/flyte/cacheservice/pkg/runtime/configs" + "github.com/flyteorg/flyte/flytestdlib/database" + "github.com/flyteorg/flyte/flytestdlib/promutils" +) + +var ( + _ interfaces.CachedOutputRepo = &memClient{} +) + +type memClient struct { + cacheMap *map[string]*models.CachedOutput +} + +// Get returns the cached output for the given key. It returns an error if the key does not exist. +func (c *memClient) Get(ctx context.Context, key string) (*models.CachedOutput, error) { + cache := *c.cacheMap + if value, exists := cache[key]; exists { + return value, nil + } + + return nil, errors.NewNotFoundError("output", key) +} + +// Put will always set the value for the given key. It will overwrite the existing value if it exists. +func (c *memClient) Put(ctx context.Context, key string, cachedOutput *models.CachedOutput) error { + cache := *c.cacheMap + cache[key] = cachedOutput + return nil +} + +// Delete is an idempotent operation. It will not return an error if the key does not exist. +func (c *memClient) Delete(ctx context.Context, key string) error { + cache := *c.cacheMap + if _, exists := cache[key]; exists { + delete(cache, key) + return nil + } + + return errors.NewNotFoundError("output", key) +} + +var ( + _ interfaces.ReservationRepo = &reservationMemClient{} +) + +type reservationMemClient struct { + keyValStore *map[string]*models.CacheReservation +} + +func (r reservationMemClient) Create(ctx context.Context, reservation *models.CacheReservation, now time.Time) error { + reservationStore := *r.keyValStore + if _, exists := reservationStore[reservation.Key]; exists { + return errors.NewCacheServiceErrorf(codes.AlreadyExists, "reservation with key %s already exists", reservation.Key) + } + reservationStore[reservation.Key] = reservation + return nil +} + +func (r reservationMemClient) Update(ctx context.Context, reservation *models.CacheReservation, now time.Time) error { + reservationStore := *r.keyValStore + reservationStore[reservation.Key] = reservation + return nil +} + +func (r reservationMemClient) Get(ctx context.Context, key string) (*models.CacheReservation, error) { + reservationStore := *r.keyValStore + if value, exists := reservationStore[key]; exists { + return value, nil + } + + return nil, errors.NewNotFoundError("reservation", key) +} + +func (r reservationMemClient) Delete(ctx context.Context, key string, ownerID string) error { + reservationStore := *r.keyValStore + delete(reservationStore, key) + return nil +} + +type CacheRepo struct { + cachedOutputRepo interfaces.CachedOutputRepo + reservationRepo interfaces.ReservationRepo +} + +func (c CacheRepo) CachedOutputRepo() interfaces.CachedOutputRepo { + return c.cachedOutputRepo +} + +func (c CacheRepo) ReservationRepo() interfaces.ReservationRepo { + return c.reservationRepo +} + +func GetRepositories(ctx context.Context, serviceConfig configs.CacheServiceConfig, dbConfig database.DbConfig, scope promutils.Scope) interfaces.RepositoryInterface { + outputClientType := serviceConfig.OutputDataStoreType + reservationClientType := serviceConfig.ReservationDataStoreType + + var postgresConnection *gorm.DB + //var postgresRepoMetrics postgres.GormMetrics + if outputClientType == configs.Postgres || reservationClientType == configs.Postgres { + var err error + dbHandle, err := postgres.NewDBHandle(ctx, &dbConfig) + if err != nil { + panic(err) + } + postgresConnection = dbHandle.DB + + //postgresRepoMetrics = postgres.NewPostgresRepoMetrics(scope.NewSubScope("postgres")) + } + + var outputRepo interfaces.CachedOutputRepo + switch outputClientType { + case configs.Mem: + cacheMap := make(map[string]*models.CachedOutput) + outputRepo = &memClient{cacheMap: &cacheMap} + case configs.Postgres: + outputRepo = postgres.NewPostgresOutputRepo(postgresConnection, scope.NewSubScope("repositories")) + default: + panic("Invalid output data store type") + } + + var reservationRepo interfaces.ReservationRepo + switch reservationClientType { + case configs.Mem: + keyValStore := make(map[string]*models.CacheReservation) + reservationRepo = &reservationMemClient{keyValStore: &keyValStore} + case configs.Postgres: + reservationRepo = postgres.NewReservationRepo(postgresConnection, scope.NewSubScope("repositories")) + default: + panic("Invalid reservation data store type") + } + + return &CacheRepo{ + cachedOutputRepo: outputRepo, + reservationRepo: reservationRepo, + } +} diff --git a/cacheservice/repositories/transformers/cached_output.go b/cacheservice/pkg/repositories/transformers/cached_output.go similarity index 97% rename from cacheservice/repositories/transformers/cached_output.go rename to cacheservice/pkg/repositories/transformers/cached_output.go index d6894d4c25..48f74a628f 100644 --- a/cacheservice/repositories/transformers/cached_output.go +++ b/cacheservice/pkg/repositories/transformers/cached_output.go @@ -7,7 +7,7 @@ import ( "google.golang.org/grpc/codes" "github.com/flyteorg/flyte/cacheservice/pkg/errors" - "github.com/flyteorg/flyte/cacheservice/repositories/models" + "github.com/flyteorg/flyte/cacheservice/pkg/repositories/models" "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/cacheservice" "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/core" "github.com/flyteorg/flyte/flytestdlib/logger" @@ -38,7 +38,9 @@ func CreateCachedOutputModel(ctx context.Context, key string, cachedOutput *cach } return &models.CachedOutput{ - ID: key, + BaseModel: models.BaseModel{ + ID: key, + }, OutputURI: cachedOutput.GetOutputUri(), OutputLiteral: outputLiteralBytes, Identifier: models.Identifier{ diff --git a/cacheservice/repositories/transformers/cached_output_test.go b/cacheservice/pkg/repositories/transformers/cached_output_test.go similarity index 94% rename from cacheservice/repositories/transformers/cached_output_test.go rename to cacheservice/pkg/repositories/transformers/cached_output_test.go index 185b0e9074..978ab701ad 100644 --- a/cacheservice/repositories/transformers/cached_output_test.go +++ b/cacheservice/pkg/repositories/transformers/cached_output_test.go @@ -8,7 +8,7 @@ import ( "github.com/stretchr/testify/assert" "google.golang.org/grpc/codes" - "github.com/flyteorg/flyte/cacheservice/repositories/models" + "github.com/flyteorg/flyte/cacheservice/pkg/repositories/models" "github.com/flyteorg/flyte/flyteidl/clients/go/coreutils" "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/cacheservice" "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/core" @@ -63,7 +63,9 @@ func TestCreateCachedOutputModel(t *testing.T) { cachedOutput: validOutputLiterals, expectError: false, expectedOutput: &models.CachedOutput{ - ID: sampleKey, + BaseModel: models.BaseModel{ + ID: sampleKey, + }, OutputURI: "", OutputLiteral: sampleOutputLiteralBytes, Identifier: models.Identifier{}, @@ -76,7 +78,9 @@ func TestCreateCachedOutputModel(t *testing.T) { cachedOutput: validOutputKeyMap, expectError: false, expectedOutput: &models.CachedOutput{ - ID: sampleKey, + BaseModel: models.BaseModel{ + ID: sampleKey, + }, OutputURI: sampleURI, OutputLiteral: nil, Identifier: models.Identifier{}, diff --git a/cacheservice/repositories/transformers/reservation.go b/cacheservice/pkg/repositories/transformers/reservation.go similarity index 77% rename from cacheservice/repositories/transformers/reservation.go rename to cacheservice/pkg/repositories/transformers/reservation.go index 0e6ac30f9d..4f26bf97fd 100644 --- a/cacheservice/repositories/transformers/reservation.go +++ b/cacheservice/pkg/repositories/transformers/reservation.go @@ -5,11 +5,11 @@ import ( "google.golang.org/protobuf/types/known/timestamppb" - "github.com/flyteorg/flyte/cacheservice/repositories/models" + "github.com/flyteorg/flyte/cacheservice/pkg/repositories/models" "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/cacheservice" ) -func FromReservationModel(ctx context.Context, reservationModel *models.Reservation) *cacheservice.Reservation { +func FromReservationModel(ctx context.Context, reservationModel *models.CacheReservation) *cacheservice.Reservation { return &cacheservice.Reservation{ Key: reservationModel.Key, OwnerId: reservationModel.OwnerID, diff --git a/cacheservice/repositories/transformers/reservation_test.go b/cacheservice/pkg/repositories/transformers/reservation_test.go similarity index 84% rename from cacheservice/repositories/transformers/reservation_test.go rename to cacheservice/pkg/repositories/transformers/reservation_test.go index b26d853aab..8ab5e0c243 100644 --- a/cacheservice/repositories/transformers/reservation_test.go +++ b/cacheservice/pkg/repositories/transformers/reservation_test.go @@ -7,7 +7,7 @@ import ( "github.com/stretchr/testify/assert" - "github.com/flyteorg/flyte/cacheservice/repositories/models" + "github.com/flyteorg/flyte/cacheservice/pkg/repositories/models" ) func TestFromReservationModel(t *testing.T) { @@ -17,7 +17,7 @@ func TestFromReservationModel(t *testing.T) { testOwnerID := "test-owner-id" testTime := time.Date(2024, time.January, 1, 0, 0, 0, 0, time.UTC) - mockReservationModel := &models.Reservation{ + mockReservationModel := &models.CacheReservation{ Key: testKey, OwnerID: testOwnerID, ExpiresAt: testTime, diff --git a/cacheservice/pkg/rpc/cacheservice/service.go b/cacheservice/pkg/rpc/cacheservice/service.go index 93c619e4d7..f4c9d93c07 100644 --- a/cacheservice/pkg/rpc/cacheservice/service.go +++ b/cacheservice/pkg/rpc/cacheservice/service.go @@ -18,6 +18,7 @@ import ( "github.com/flyteorg/flyte/cacheservice/pkg/config" "github.com/flyteorg/flyte/cacheservice/pkg/manager/impl" "github.com/flyteorg/flyte/cacheservice/pkg/manager/interfaces" + "github.com/flyteorg/flyte/cacheservice/pkg/repositories" "github.com/flyteorg/flyte/cacheservice/pkg/runtime" "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/cacheservice" "github.com/flyteorg/flyte/flytestdlib/contextutils" @@ -80,12 +81,13 @@ func NewCacheServiceServer() *CacheService { panic(err) } - outputStore := impl.NewCacheOutputStore(dataStorageClient, storagePrefix) - dataStore := impl.NewCacheDataStore(ctx, cacheServiceConfig) - reservationStore := impl.NewReservationDataStore(ctx, cacheServiceConfig) + outputStore := impl.NewCacheOutputBlobStore(dataStorageClient, storagePrefix) + // TODO - @pvditt this should prolly not be handled here + pgDbConfigValues := configProvider.ApplicationConfiguration().GetDbConfig() + repos := repositories.GetRepositories(ctx, cacheServiceConfig, *pgDbConfigValues, cacheServiceScope) return &CacheService{ - CacheManager: impl.NewCacheManager(outputStore, dataStore, reservationStore, cacheServiceConfig.MaxInlineSizeBytes, cacheServiceScope.NewSubScope("cache"), + CacheManager: impl.NewCacheManager(outputStore, repos.CachedOutputRepo(), repos.ReservationRepo(), cacheServiceConfig.MaxInlineSizeBytes, cacheServiceScope.NewSubScope("cache"), time.Duration(cacheServiceConfig.HeartbeatGracePeriodMultiplier), cacheServiceConfig.MaxReservationHeartbeat.Duration), } } diff --git a/cacheservice/pkg/runtime/application_config_provider.go b/cacheservice/pkg/runtime/application_config_provider.go index 7df01775e2..ee236fefc2 100644 --- a/cacheservice/pkg/runtime/application_config_provider.go +++ b/cacheservice/pkg/runtime/application_config_provider.go @@ -1,17 +1,25 @@ package runtime import ( + "context" + "io/ioutil" + "os" + "strings" + "github.com/flyteorg/flyte/cacheservice/pkg/runtime/configs" "github.com/flyteorg/flyte/flytestdlib/config" + "github.com/flyteorg/flyte/flytestdlib/database" + "github.com/flyteorg/flyte/flytestdlib/logger" ) const cacheservice = "cacheservice" var cacheserviceConfig = config.MustRegisterSection(cacheservice, &configs.CacheServiceConfig{}) -// Defines the interface to return top-level config structs necessary to start up a cacheservice application. +// ApplicationConfiguration defines the interface to return top-level config structs necessary to start up a cacheservice application. type ApplicationConfiguration interface { GetCacheServiceConfig() configs.CacheServiceConfig + GetDbConfig() *database.DbConfig } type ApplicationConfigurationProvider struct{} @@ -20,6 +28,26 @@ func (p *ApplicationConfigurationProvider) GetCacheServiceConfig() configs.Cache return *cacheserviceConfig.GetConfig().(*configs.CacheServiceConfig) } +func (p *ApplicationConfigurationProvider) GetDbConfig() *database.DbConfig { + dbConfigSection := database.GetConfig() + if len(dbConfigSection.Postgres.PasswordPath) > 0 { + if _, err := os.Stat(dbConfigSection.Postgres.PasswordPath); os.IsNotExist(err) { + logger.Fatalf(context.Background(), + "missing database password at specified path [%s]", dbConfigSection.Postgres.PasswordPath) + } + passwordVal, err := ioutil.ReadFile(dbConfigSection.Postgres.PasswordPath) + if err != nil { + logger.Fatalf(context.Background(), "failed to read database password from path [%s] with err: %v", + dbConfigSection.Postgres.PasswordPath, err) + } + // Passwords can contain special characters as long as they are percent encoded + // https://www.postgresql.org/docs/current/libpq-connect.html + dbConfigSection.Postgres.Password = strings.TrimSpace(string(passwordVal)) + } + + return dbConfigSection +} + func NewApplicationConfigurationProvider() ApplicationConfiguration { return &ApplicationConfigurationProvider{} } diff --git a/cacheservice/pkg/runtime/configs/cache_service_config.go b/cacheservice/pkg/runtime/configs/cache_service_config.go index 40c990c913..4c3e8b9b8c 100644 --- a/cacheservice/pkg/runtime/configs/cache_service_config.go +++ b/cacheservice/pkg/runtime/configs/cache_service_config.go @@ -10,8 +10,7 @@ type DataStoreType = string const ( Mem DataStoreType = "mem" - DynamoDB DataStoreType = "dynamodb" - Redis DataStoreType = "redis" + Postgres DataStoreType = "postgres" ) //go:generate pflags CacheServiceConfig --default-var=defaultConfig @@ -22,8 +21,8 @@ var defaultConfig = &CacheServiceConfig{ ProfilerPort: 10254, HeartbeatGracePeriodMultiplier: 3, MaxReservationHeartbeat: config.Duration{Duration: time.Second * 10}, - DataStoreType: Mem, - ReservationDataStoreType: Mem, + OutputDataStoreType: Postgres, + ReservationDataStoreType: Postgres, MaxInlineSizeBytes: 0, AwsRegion: "us-west-2", RedisAddress: "localhost:6379", @@ -38,7 +37,7 @@ type CacheServiceConfig struct { ProfilerPort int `json:"profiler-port" pflag:",Port that the profiling service is listening on."` HeartbeatGracePeriodMultiplier int `json:"heartbeat-grace-period-multiplier" pflag:",Number of heartbeats before a reservation expires without an extension."` MaxReservationHeartbeat config.Duration `json:"max-reservation-heartbeat" pflag:",The maximum available reservation extension heartbeat interval."` - DataStoreType DataStoreType `json:"data-store-type" pflag:",Cache storage implementation to use"` + OutputDataStoreType DataStoreType `json:"data-store-type" pflag:",Cache storage implementation to use"` ReservationDataStoreType DataStoreType `json:"reservation-data-store-type" pflag:",Reservation storage implementation to use"` MaxInlineSizeBytes int64 `json:"maxInlineSizeBytes" pflag:",The maximum size that an output literal will be stored in line. Default 0 means everything will be offloaded to blob storage."` AwsRegion string `json:"aws-region" pflag:",Region to connect to."` diff --git a/cacheservice/pkg/runtime/configs/cacheserviceconfig_flags.go b/cacheservice/pkg/runtime/configs/cacheserviceconfig_flags.go index 2414e77bf0..be11a8cdac 100755 --- a/cacheservice/pkg/runtime/configs/cacheserviceconfig_flags.go +++ b/cacheservice/pkg/runtime/configs/cacheserviceconfig_flags.go @@ -55,7 +55,7 @@ func (cfg CacheServiceConfig) GetPFlagSet(prefix string) *pflag.FlagSet { cmdFlags.Int(fmt.Sprintf("%v%v", prefix, "profiler-port"), defaultConfig.ProfilerPort, "Port that the profiling service is listening on.") cmdFlags.Int(fmt.Sprintf("%v%v", prefix, "heartbeat-grace-period-multiplier"), defaultConfig.HeartbeatGracePeriodMultiplier, "Number of heartbeats before a reservation expires without an extension.") cmdFlags.String(fmt.Sprintf("%v%v", prefix, "max-reservation-heartbeat"), defaultConfig.MaxReservationHeartbeat.String(), "The maximum available reservation extension heartbeat interval.") - cmdFlags.String(fmt.Sprintf("%v%v", prefix, "data-store-type"), defaultConfig.DataStoreType, "Cache storage implementation to use") + cmdFlags.String(fmt.Sprintf("%v%v", prefix, "data-store-type"), defaultConfig.OutputDataStoreType, "Cache storage implementation to use") cmdFlags.String(fmt.Sprintf("%v%v", prefix, "reservation-data-store-type"), defaultConfig.ReservationDataStoreType, "Reservation storage implementation to use") cmdFlags.Int64(fmt.Sprintf("%v%v", prefix, "maxInlineSizeBytes"), defaultConfig.MaxInlineSizeBytes, "The maximum size that an output literal will be stored in line. Default 0 means everything will be offloaded to blob storage.") cmdFlags.String(fmt.Sprintf("%v%v", prefix, "aws-region"), defaultConfig.AwsRegion, "Region to connect to.") diff --git a/cacheservice/pkg/runtime/configs/cacheserviceconfig_flags_test.go b/cacheservice/pkg/runtime/configs/cacheserviceconfig_flags_test.go index 83fbfd5335..b1e5bc0a97 100755 --- a/cacheservice/pkg/runtime/configs/cacheserviceconfig_flags_test.go +++ b/cacheservice/pkg/runtime/configs/cacheserviceconfig_flags_test.go @@ -176,7 +176,7 @@ func TestCacheServiceConfig_SetFlags(t *testing.T) { cmdFlags.Set("data-store-type", testValue) if vString, err := cmdFlags.GetString("data-store-type"); err == nil { - testDecodeJson_CacheServiceConfig(t, fmt.Sprintf("%v", vString), &actual.DataStoreType) + testDecodeJson_CacheServiceConfig(t, fmt.Sprintf("%v", vString), &actual.OutputDataStoreType) } else { assert.FailNow(t, err.Error()) diff --git a/cacheservice/repositories/models/cached_output.go b/cacheservice/repositories/models/cached_output.go deleted file mode 100644 index 75937b8896..0000000000 --- a/cacheservice/repositories/models/cached_output.go +++ /dev/null @@ -1,20 +0,0 @@ -package models - -import "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/core" - -type Identifier struct { - ResourceType core.ResourceType `dynamodbav:"resourceType"` - Project string `dynamodbav:"project"` - Domain string `dynamodbav:"domain"` - Name string `dynamodbav:"name"` - Version string `dynamodbav:"version"` - Org string `dynamodbav:"org"` -} - -type CachedOutput struct { - ID string `dynamodbav:"id"` - OutputURI string `dynamodbav:"outputUri"` - OutputLiteral []byte `dynamodbav:"outputLiteral"` - SerializedMetadata []byte `dynamodbav:"keyMap"` - Identifier Identifier `dynamodbav:"identifier"` -} diff --git a/cmd/single/start.go b/cmd/single/start.go index 7f1add5ed6..07973dab25 100644 --- a/cmd/single/start.go +++ b/cmd/single/start.go @@ -17,6 +17,7 @@ import ( ctrlWebhook "sigs.k8s.io/controller-runtime/pkg/webhook" cacheserviceConfig "github.com/flyteorg/flyte/cacheservice/pkg/config" + cacheserviceRepo "github.com/flyteorg/flyte/cacheservice/pkg/repositories" "github.com/flyteorg/flyte/cacheservice/pkg/rpc/cacheservice" datacatalogConfig "github.com/flyteorg/flyte/datacatalog/pkg/config" datacatalogRepo "github.com/flyteorg/flyte/datacatalog/pkg/repositories" @@ -54,6 +55,10 @@ func startDataCatalog(ctx context.Context, _ DataCatalog) error { } func startCacheService(ctx context.Context, _ CacheService) error { + if err := cacheserviceRepo.Migrate(ctx); err != nil { + return err + } + cacheCfg := cacheserviceConfig.GetConfig() return cacheservice.ServeInsecure(ctx, cacheCfg) } @@ -206,8 +211,9 @@ var startCmd = &cobra.Command{ cfg := GetConfig() for _, serviceName := range []string{otelutils.AdminClientTracer, otelutils.AdminGormTracer, otelutils.AdminServerTracer, - otelutils.BlobstoreClientTracer, otelutils.CacheServiceClientTracer, otelutils.DataCatalogClientTracer, - otelutils.DataCatalogGormTracer, otelutils.DataCatalogServerTracer, otelutils.FlytePropellerTracer, otelutils.K8sClientTracer} { + otelutils.BlobstoreClientTracer, otelutils.CacheServiceClientTracer, otelutils.CacheServiceGormTracer, + otelutils.CacheServiceServerTracer, otelutils.DataCatalogClientTracer, otelutils.DataCatalogGormTracer, + otelutils.DataCatalogServerTracer, otelutils.FlytePropellerTracer, otelutils.K8sClientTracer} { if err := otelutils.RegisterTracerProvider(serviceName, otelutils.GetConfig()); err != nil { logger.Errorf(ctx, "Failed to create otel tracer provider. %v", err) return err diff --git a/flyte-single-binary-local.yaml b/flyte-single-binary-local.yaml index ca63458b03..bf6f75b560 100644 --- a/flyte-single-binary-local.yaml +++ b/flyte-single-binary-local.yaml @@ -12,8 +12,9 @@ flyteadmin: catalog-cache: endpoint: localhost:8081 + cache-endpoint: localhost:8091 insecure: true - type: datacatalog + type: cacheservice cluster_resources: standaloneDeployment: false diff --git a/go.mod b/go.mod index 2c6e9e5767..3346ffbeae 100644 --- a/go.mod +++ b/go.mod @@ -32,22 +32,19 @@ require ( github.com/AzureAD/microsoft-authentication-library-for-go v1.2.0 // indirect github.com/GoogleCloudPlatform/spark-on-k8s-operator v0.0.0-20200723154620-6f35a1152625 // indirect github.com/NYTimes/gizmo v1.3.6 // indirect + github.com/Selvatico/go-mocket v1.0.7 // indirect github.com/Shopify/sarama v1.26.4 // indirect github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect github.com/aws/aws-sdk-go v1.46.2 // indirect github.com/aws/aws-sdk-go-v2 v1.24.1 // indirect github.com/aws/aws-sdk-go-v2/config v1.26.6 // indirect github.com/aws/aws-sdk-go-v2/credentials v1.16.16 // indirect - github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.12.17 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.11 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.10 // indirect github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.10 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.7.3 // indirect github.com/aws/aws-sdk-go-v2/service/athena v1.0.0 // indirect - github.com/aws/aws-sdk-go-v2/service/dynamodb v1.27.1 // indirect - github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.18.7 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.8.11 // indirect github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.10 // indirect github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.26.1 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.18.7 // indirect @@ -68,7 +65,6 @@ require ( github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect github.com/dgraph-io/ristretto v0.1.1 // indirect - github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/eapache/go-resiliency v1.2.0 // indirect github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 // indirect @@ -91,7 +87,6 @@ require ( github.com/go-openapi/jsonreference v0.20.2 // indirect github.com/go-openapi/swag v0.22.4 // indirect github.com/go-redis/redis v6.15.7+incompatible // indirect - github.com/go-redis/redis/v8 v8.11.5 // indirect github.com/go-test/deep v1.1.0 // indirect github.com/goccy/go-json v0.10.2 // indirect github.com/gofrs/uuid v4.4.0+incompatible // indirect diff --git a/go.sum b/go.sum index 2598167994..e97e4e77f5 100644 --- a/go.sum +++ b/go.sum @@ -122,8 +122,6 @@ github.com/aws/aws-sdk-go-v2/config v1.26.6 h1:Z/7w9bUqlRI0FFQpetVuFYEsjzE3h7fpU github.com/aws/aws-sdk-go-v2/config v1.26.6/go.mod h1:uKU6cnDmYCvJ+pxO9S4cWDb2yWWIH5hra+32hVh1MI4= github.com/aws/aws-sdk-go-v2/credentials v1.16.16 h1:8q6Rliyv0aUFAVtzaldUEcS+T5gbadPbWdV1WcAddK8= github.com/aws/aws-sdk-go-v2/credentials v1.16.16/go.mod h1:UHVZrdUsv63hPXFo1H7c5fEneoVo9UXiz36QG1GEPi0= -github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.12.17 h1:jPuObStSZU1cGheSslAbF2nA4c/IgeIQA1X9frB60Oc= -github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.12.17/go.mod h1:df3uvEupLM3MkLim3BDkCaRpgAROW7wk41dwNQjw0kA= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.11 h1:c5I5iH+DZcH3xOIMlz3/tCKJDaHFwYEmxvlh2fAcFo8= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.11/go.mod h1:cRrYDYAMUohBJUtUnOhydaMHtiK/1NZ0Otc9lIb6O0Y= github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.10 h1:vF+Zgd9s+H4vOXd5BMaPWykta2a6Ih0AKLq/X6NYKn4= @@ -134,14 +132,8 @@ github.com/aws/aws-sdk-go-v2/internal/ini v1.7.3 h1:n3GDfwqF2tzEkXlv5cuy4iy7LpKD github.com/aws/aws-sdk-go-v2/internal/ini v1.7.3/go.mod h1:6fQQgfuGmw8Al/3M2IgIllycxV7ZW7WCdVSqfBeUiCY= github.com/aws/aws-sdk-go-v2/service/athena v1.0.0 h1:UfrZP3NMTTKpOsf/P8uCaOxz3U2CNGEizdQKcObY7Ds= github.com/aws/aws-sdk-go-v2/service/athena v1.0.0/go.mod h1:qY8QFbemf2ceqweXcS6hQqiiIe1z42WqTvHsK2Lb0rE= -github.com/aws/aws-sdk-go-v2/service/dynamodb v1.27.1 h1:plNo3WtooT2fYnhdyuzzsIJ4QWzcF5AT9oFbnrYC5Dw= -github.com/aws/aws-sdk-go-v2/service/dynamodb v1.27.1/go.mod h1:N5tqZcYMM0N1PN7UQYJNWuGyO886OfnMhf/3MAbqMcI= -github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.18.7 h1:srShyROqxzC7p18Ws8mqM2sqxJO/8L3Kpiqf+NboJLg= -github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.18.7/go.mod h1:9efZgg4nJCGRp91MuHhkwd2kvyp7PWLRYYk5WjEQ5ts= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4 h1:/b31bi3YVNlkzkBrm9LfpaKoaYZUxIAj4sHfOTmLfqw= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4/go.mod h1:2aGXHFmbInwgP9ZfpmdIfOELL79zhdNYNmReK8qDfdQ= -github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.8.11 h1:e9AVb17H4x5FTE5KWIP5M1Du+9M86pS+Hw0lBUdN8EY= -github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.8.11/go.mod h1:B90ZQJa36xo0ph9HsoteI1+r8owgQH/U1QNfqZQkj1Q= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.10 h1:DBYTXwIGQSGs9w4jKm60F5dmCQ3EEruxdc0MFh+3EY4= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.10/go.mod h1:wohMUQiFdzo0NtxbBg0mSRGZ4vL3n0dKjLTINdcIino= github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.26.1 h1:Sn3MAV9YeACCULaxNWWYFH1a6G4wYFwBn3/TA5MwE2Q= @@ -243,8 +235,6 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZm github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WAFKLNi6ZS0675eEUC9y3AlwSbQu1Y= github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= -github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= -github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= @@ -399,8 +389,6 @@ github.com/go-openapi/validate v0.19.3/go.mod h1:90Vh6jjkTn+OT1Eefm0ZixWNFjhtOH7 github.com/go-openapi/validate v0.19.10/go.mod h1:RKEZTUWDkxKQxN2jDT7ZnZi2bhZlbNMAuKvKB+IaGx8= github.com/go-redis/redis v6.15.7+incompatible h1:3skhDh95XQMpnqeqNftPkQD9jL9e5e36z/1SUm6dy1U= github.com/go-redis/redis v6.15.7+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= -github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI= -github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=