diff --git a/Makefile b/Makefile index 789e60db..6638beb6 100644 --- a/Makefile +++ b/Makefile @@ -119,6 +119,10 @@ lint: gen test: gen go test ./internal/... +.PHONY: test-nocache +test-nocache: gen + go test ./internal/... -count=1 + .PHONY: run/migrate run/migrate: gen go run main.go migrate --logtostderr=true -m config/metadata-library diff --git a/go.mod b/go.mod index 96c0acaa..e96d7555 100644 --- a/go.mod +++ b/go.mod @@ -12,8 +12,10 @@ require ( github.com/spf13/cobra v1.7.0 github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.16.0 + github.com/stretchr/testify v1.8.4 + github.com/testcontainers/testcontainers-go v0.25.0 github.com/vektah/gqlparser/v2 v2.5.8 - golang.org/x/sync v0.2.0 + golang.org/x/sync v0.3.0 google.golang.org/grpc v1.57.0 google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.3.0 google.golang.org/protobuf v1.31.0 @@ -23,29 +25,73 @@ require ( ) require ( + github.com/cucumber/gherkin/go/v26 v26.2.0 // indirect + github.com/cucumber/messages/go/v21 v21.0.1 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/gofrs/uuid v4.3.1+incompatible // indirect + github.com/hashicorp/go-immutable-radix v1.3.1 // indirect + github.com/hashicorp/go-memdb v1.3.4 // indirect + github.com/hashicorp/golang-lru v0.5.4 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect +) + +require ( + dario.cat/mergo v1.0.0 // indirect + github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect + github.com/Microsoft/go-winio v0.6.1 // indirect + github.com/Microsoft/hcsshim v0.11.0 // indirect github.com/agnivade/levenshtein v1.1.1 // indirect + github.com/cenkalti/backoff/v4 v4.2.1 // indirect + github.com/containerd/containerd v1.7.6 // indirect + github.com/cpuguy83/dockercfg v0.3.1 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect + github.com/cucumber/godog v0.13.0 + github.com/docker/distribution v2.8.2+incompatible // indirect + github.com/docker/docker v24.0.6+incompatible // indirect + github.com/docker/go-connections v0.4.0 // indirect + github.com/docker/go-units v0.5.0 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect + github.com/go-ole/go-ole v1.2.6 // indirect + github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.3 // indirect + github.com/google/uuid v1.3.1 // indirect github.com/gorilla/websocket v1.5.0 // indirect github.com/hashicorp/golang-lru/v2 v2.0.3 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect + github.com/klauspost/compress v1.16.0 // indirect + github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mattn/go-sqlite3 v1.14.17 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/moby/patternmatcher v0.5.0 // indirect + github.com/moby/sys/sequential v0.5.0 // indirect + github.com/moby/term v0.5.0 // indirect + github.com/morikuni/aec v1.0.0 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.1.0-rc4 // indirect + github.com/opencontainers/runc v1.1.5 // indirect github.com/pelletier/go-toml/v2 v2.0.8 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/searKing/golang/go v1.2.52 // indirect github.com/searKing/golang/tools v1.2.29 // indirect + github.com/shirou/gopsutil/v3 v3.23.8 // indirect + github.com/shoenig/go-m1cpu v0.1.6 // indirect + github.com/sirupsen/logrus v1.9.0 // indirect github.com/spf13/afero v1.9.5 // indirect github.com/spf13/cast v1.5.1 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/subosito/gotenv v1.4.2 // indirect + github.com/tklauser/go-sysconf v0.3.12 // indirect + github.com/tklauser/numcpus v0.6.1 // indirect github.com/urfave/cli/v2 v2.25.5 // indirect github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect + github.com/yusufpapurcu/wmi v1.2.3 // indirect + golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 // indirect golang.org/x/mod v0.10.0 // indirect golang.org/x/net v0.17.0 // indirect golang.org/x/sys v0.13.0 // indirect diff --git a/go.sum b/go.sum index 938301e1..04a12840 100644 --- a/go.sum +++ b/go.sum @@ -35,38 +35,77 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= +dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= +dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/99designs/gqlgen v0.17.36 h1:u/o/rv2SZ9s5280dyUOOrkpIIkr/7kITMXYD3rkJ9go= github.com/99designs/gqlgen v0.17.36/go.mod h1:6RdyY8puhCoWAQVr2qzF2OMVfudQzc8ACxzpzluoQm4= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 h1:bvDV9vkmnHYOMsOr4WLk+Vo07yKIzd94sVoIqshQ4bU= +github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= +github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= +github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= +github.com/Microsoft/hcsshim v0.11.0 h1:7EFNIY4igHEXUdj1zXgAyU3fLc7QfOKHbkldRVTBdiM= +github.com/Microsoft/hcsshim v0.11.0/go.mod h1:OEthFdQv/AD2RAdzR6Mm1N1KPCztGKDurW1Z8b8VGMM= github.com/agnivade/levenshtein v1.1.1 h1:QY8M92nrzkmr798gCo3kmMyqXFzdQVpxLlGPRBij0P8= github.com/agnivade/levenshtein v1.1.1/go.mod h1:veldBMzWxcCG2ZvUTKD2kJNRdCk5hVbJomOvKkmgYbo= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883 h1:bvNMNQO63//z+xNgfBlViaCIJKLlCJ6/fmUseuG0wVQ= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0 h1:jfIu9sQUG6Ig+0+Ap1h4unLjW6YQJpKZVmUzxsD4E/Q= github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0/go.mod h1:t2tdKJDJF9BV14lnkjHmOQgcvEKgtqs5a1N3LNdJhGE= +github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= +github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/checkpoint-restore/go-criu/v5 v5.3.0/go.mod h1:E/eQpaFtUKGOOSEBZgmKAcn+zUUwWxqcaKZlF54wK8E= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/cilium/ebpf v0.7.0/go.mod h1:/oI2+1shJiTGAMgl6/RgJr36Eo1jzrRcAWbcXO2usCA= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U= +github.com/containerd/containerd v1.7.6 h1:oNAVsnhPoy4BTPQivLgTzI9Oleml9l/+eYIDYXRCYo8= +github.com/containerd/containerd v1.7.6/go.mod h1:SY6lrkkuJT40BVNO37tlYTSnKJnP5AXBc0fhx0q+TJ4= +github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/cpuguy83/dockercfg v0.3.1 h1:/FpZ+JaygUR/lZP2NlFI2DVfrOEMAIKP5wWEJdoYe9E= +github.com/cpuguy83/dockercfg v0.3.1/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= +github.com/cucumber/gherkin/go/v26 v26.2.0 h1:EgIjePLWiPeslwIWmNQ3XHcypPsWAHoMCz/YEBKP4GI= +github.com/cucumber/gherkin/go/v26 v26.2.0/go.mod h1:t2GAPnB8maCT4lkHL99BDCVNzCh1d7dBhCLt150Nr/0= +github.com/cucumber/godog v0.13.0 h1:KvX9kNWmAJwp882HmObGOyBbNUP5SXQ+SDLNajsuV7A= +github.com/cucumber/godog v0.13.0/go.mod h1:FX3rzIDybWABU4kuIXLZ/qtqEe1Ac5RdXmqvACJOces= +github.com/cucumber/messages/go/v21 v21.0.1 h1:wzA0LxwjlWQYZd32VTlAVDTkW6inOFmSM+RuOwHZiMI= +github.com/cucumber/messages/go/v21 v21.0.1/go.mod h1:zheH/2HS9JLVFukdrsPWoPdmUtmYQAQPLk7w5vWsk5s= +github.com/cucumber/messages/go/v22 v22.0.0/go.mod h1:aZipXTKc0JnjCsXrJnuZpWhtay93k7Rn3Dee7iyPJjs= +github.com/cyphar/filepath-securejoin v0.2.3/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgryski/trifles v0.0.0-20200323201526-dd97f9abfb48 h1:fRzb/w+pyskVMQ+UbP35JkH8yB7MYb4q/qhBarqZE6g= github.com/dgryski/trifles v0.0.0-20200323201526-dd97f9abfb48/go.mod h1:if7Fbed8SFyPtHLHbg49SI7NAdJiC5WIA09pe59rfAA= +github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8= +github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= +github.com/docker/docker v24.0.6+incompatible h1:hceabKCtUgDqPu+qm0NgsaXf28Ljf4/pWFL7xjWWDgE= +github.com/docker/docker v24.0.6+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= +github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= +github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= +github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= @@ -75,6 +114,15 @@ github.com/go-chi/chi/v5 v5.0.10/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNIT github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/godbus/dbus/v5 v5.0.6/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gofrs/uuid v4.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/gofrs/uuid v4.3.1+incompatible h1:0/KbAdpx3UXAx1kEOWHJeOkpbgRFGHVgv+CFIY7dBJI= +github.com/gofrs/uuid v4.3.1+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v1.1.0 h1:/d3pCKDPWNnvIWe0vVUpNP32qc8U3PDVxySP/y360qE= github.com/golang/glog v1.1.0/go.mod h1:pfYeQZ3JWZoXTV5sFc986z3HTpwQs9At6P4ImfuP3NQ= @@ -117,7 +165,9 @@ github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= @@ -133,6 +183,8 @@ github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= +github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= @@ -140,8 +192,18 @@ github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWm github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0 h1:2cz5kSrxzMYHiWOBbKj8itQm+nRykkB8aMv4ThcHYHA= github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0/go.mod h1:w9Y7gY31krpLmrVU5ZPG9H7l9fZuRu5/3R3S3FMtVQ4= +github.com/hashicorp/go-immutable-radix v1.3.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-immutable-radix v1.3.1 h1:DKHmCUm2hRBK510BaiZlwvpD40f8bJFeZnpfm2KLowc= +github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-memdb v1.3.4 h1:XSL3NR682X/cVk2IeV0d70N4DZ9ljI885xAEU8IoK3c= +github.com/hashicorp/go-memdb v1.3.4/go.mod h1:uBTr1oQbtuMgd1SSGoR8YV27eT3sBHbYiNm53bMpgSg= +github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.2 h1:cfejS+Tpcp13yd5nYHWDI6qVCny6wyX2Mt5SGur2IGE= +github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= +github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/golang-lru/v2 v2.0.3 h1:kmRrRLlInXvng0SmLxmQpQkpbYAvcXm7NPDrgxJa9mE= github.com/hashicorp/golang-lru/v2 v2.0.3/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= @@ -156,28 +218,56 @@ github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.16.0 h1:iULayQNOReoYUe+1qtKOqw9CwJv3aNQu8ivo7lw1HU4= +github.com/klauspost/compress v1.16.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mattn/go-sqlite3 v1.14.17 h1:mCRHCLDUBXgpKAqIKsaAaAsrAlbkeomtRFKXh2L6YIM= github.com/mattn/go-sqlite3 v1.14.17/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/moby/patternmatcher v0.5.0 h1:YCZgJOeULcxLw1Q+sVR636pmS7sPEn1Qo2iAN6M7DBo= +github.com/moby/patternmatcher v0.5.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= +github.com/moby/sys/mountinfo v0.5.0/go.mod h1:3bMD3Rg+zkqx8MRYPi7Pyb0Ie97QEBmdxbhnCLlSvSU= +github.com/moby/sys/sequential v0.5.0 h1:OPvI35Lzn9K04PBbCLW0g4LcFAJgHsvXsRyewg5lXtc= +github.com/moby/sys/sequential v0.5.0/go.mod h1:tH2cOOs5V9MlPiXcQzRC+eEyab644PWKGRYaaV5ZZlo= +github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= +github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= +github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= +github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= +github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.1.0-rc4 h1:oOxKUJWnFC4YGHCCMNql1x4YaDfYBTS5Y4x/Cgeo1E0= +github.com/opencontainers/image-spec v1.1.0-rc4/go.mod h1:X4pATf0uXsnn3g5aiGIsVnJBR4mxhKzfwmvK/B2NTm8= +github.com/opencontainers/runc v1.1.5 h1:L44KXEpKmfWDcS02aeGm8QNTFXTo2D+8MYGDIJ/GDEs= +github.com/opencontainers/runc v1.1.5/go.mod h1:1J5XiS+vdZ3wCyZybsuxXZWGrgSr8fFJHLXuG2PsnNg= +github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/selinux v1.10.0/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI= github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ= github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= +github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/searKing/golang/go v1.2.52 h1:Nbsr8HIS4ATL7LY1BE3SwRyEQd+UkrRymyol46sXb3s= @@ -186,8 +276,19 @@ github.com/searKing/golang/tools v1.2.29 h1:gWX4aCA1+N7CfLpNd6tmBjYtecTksZFXziqb github.com/searKing/golang/tools v1.2.29/go.mod h1:QtwFM73H1qMKlRl0p8NRWXrNaCaffoYkNHk5YMoRyOY= github.com/searKing/golang/tools/go-enum v1.2.97 h1:mX396oCnjdGqmMmwffBwpEU3L1X/7K3eDuXOOLow+TQ= github.com/searKing/golang/tools/go-enum v1.2.97/go.mod h1:gWZ/vkIPpA0nCHDMETjzKqsHPDWSPdhZeXPxf/jWyqA= +github.com/seccomp/libseccomp-golang v0.9.2-0.20220502022130-f33da4d89646/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg= github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8= github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I= +github.com/shirou/gopsutil/v3 v3.23.8 h1:xnATPiybo6GgdRoC4YoGnxXZFRc3dqQTGi73oLvvBrE= +github.com/shirou/gopsutil/v3 v3.23.8/go.mod h1:7hmCaBn+2ZwaZOr6jmPBZDfawwMGuo1id3C6aM8EDqQ= +github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= +github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= +github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= +github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= +github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/soheilhy/cmux v0.1.5 h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js= github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= github.com/spf13/afero v1.9.5 h1:stMpOSZFs//0Lv29HduCmli3GUfpFoF3Y1Q/aXj/wVM= @@ -211,20 +312,35 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/subosito/gotenv v1.4.2 h1:X1TuBLAMDFbaTAChgCBLu3DU3UPyELpnF2jjJ2cz/S8= github.com/subosito/gotenv v1.4.2/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= +github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= +github.com/testcontainers/testcontainers-go v0.25.0 h1:erH6cQjsaJrH+rJDU9qIf89KFdhK0Bft0aEZHlYC3Vs= +github.com/testcontainers/testcontainers-go v0.25.0/go.mod h1:4sC9SiJyzD1XFi59q8umTQYWxnkweEc5OjVtTUlJzqQ= +github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= +github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= +github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= +github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= +github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli/v2 v2.25.5 h1:d0NIAyhh5shGscroL7ek/Ya9QYQE0KNabJgiUinIQkc= github.com/urfave/cli/v2 v2.25.5/go.mod h1:GHupkWPMM0M/sj1a2b4wUrWBPzazNrIjouW6fmdJLxc= github.com/vektah/gqlparser/v2 v2.5.8 h1:pm6WOnGdzFOCfcQo9L3+xzW51mKrlwTEg4Wr7AH1JW4= github.com/vektah/gqlparser/v2 v2.5.8/go.mod h1:z8xXUff237NntSuH8mLFijZ+1tjV1swDbpDqjJmk6ME= +github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= +github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw= +github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -248,6 +364,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 h1:k/i9J1pBpvlfR+9QsetwPyERsqu1GIbi967PQMq3Ivc= +golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -327,8 +445,8 @@ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI= -golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -336,9 +454,13 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191115151921-52ab43148777/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -358,13 +480,22 @@ golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211116061358-0a5406a5449c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -381,6 +512,7 @@ golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= @@ -418,6 +550,7 @@ golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= @@ -426,6 +559,7 @@ golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.9.3 h1:Gn1I8+64MsuTb/HpH+LmQtNas23LhUVr3rYZ0eKuaMM= @@ -530,12 +664,14 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= @@ -549,6 +685,7 @@ gorm.io/driver/sqlite v1.5.3 h1:7/0dUgX28KAcopdfbRWWl68Rflh6osa4rDh+m51KL2g= gorm.io/driver/sqlite v1.5.3/go.mod h1:qxAuCol+2r6PannQDpOP1FP6ag3mKi4esLnB/jHed+4= gorm.io/gorm v1.25.4 h1:iyNd8fNAe8W9dvtlgeRI5zSVZPsq3OpcTu37cYcpCmw= gorm.io/gorm v1.25.4/go.mod h1:L4uxeKpfBml98NYqVqwAdmV1a2nBtAec/cf3fpucW/k= +gotest.tools/v3 v3.5.0 h1:Ljk6PdHdOhAb5aDMWXjDLMMhph+BpztA4v1QdqEW2eY= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/internal/core/api.go b/internal/core/api.go new file mode 100644 index 00000000..a88a899f --- /dev/null +++ b/internal/core/api.go @@ -0,0 +1,45 @@ +package core + +import "github.com/opendatahub-io/model-registry/internal/model/openapi" + +// Note: for convention, here we are keeping here adherence to the mlmd side +type BaseResourceId int64 + +type ListOptions struct { + PageSize *int32 + OrderBy *string + SortOrder *string + NextPageToken *string +} + +type ModelRegistryApi interface { + // REGISTERED MODEL + + // UpsertRegisteredModel create or update a registered model, the behavior follows the same + // approach used by MLMD gRPC api. If Id is provided update the entity otherwise create a new one. + UpsertRegisteredModel(registeredModel *openapi.RegisteredModel) (*openapi.RegisteredModel, error) + + GetRegisteredModelById(id *BaseResourceId) (*openapi.RegisteredModel, error) + GetRegisteredModelByParams(name *string, externalId *string) (*openapi.RegisteredModel, error) + GetRegisteredModels(listOptions ListOptions) (*openapi.RegisteredModelList, error) + + // MODEL VERSION + + // Create a new Model Version + // or update a Model Version associated to a specific RegisteredModel identified by registeredModelId parameter + UpsertModelVersion(modelVersion *openapi.ModelVersion, registeredModelId *BaseResourceId) (*openapi.ModelVersion, error) + + GetModelVersionById(id *BaseResourceId) (*openapi.ModelVersion, error) + GetModelVersionByParams(name *string, externalId *string) (*openapi.ModelVersion, error) + GetModelVersions(listOptions ListOptions, registeredModelId *BaseResourceId) (*openapi.ModelVersionList, error) + + // MODEL ARTIFACT + + // Create or update a Model Artifact associated to a specific ModelVersion + // identified by ModelArtifact.ModelVersionId + UpsertModelArtifact(modelArtifact *openapi.ModelArtifact, modelVersionId *BaseResourceId) (*openapi.ModelArtifact, error) + + GetModelArtifactById(id *BaseResourceId) (*openapi.ModelArtifact, error) + GetModelArtifactByParams(name *string, externalId *string) (*openapi.ModelArtifact, error) + GetModelArtifacts(listOptions ListOptions, modelVersionId *BaseResourceId) (*openapi.ModelArtifactList, error) +} diff --git a/internal/core/api_utils.go b/internal/core/api_utils.go new file mode 100644 index 00000000..a3a4f010 --- /dev/null +++ b/internal/core/api_utils.go @@ -0,0 +1,43 @@ +package core + +import ( + "github.com/opendatahub-io/model-registry/internal/ml_metadata/proto" +) + +func zeroIfNil[T any](input *T) T { + if input != nil { + return *input + } + return *new(T) +} + +func BuildListOperationOptions(listOptions ListOptions) (*proto.ListOperationOptions, error) { + + result := proto.ListOperationOptions{} + if listOptions.PageSize != nil { + result.MaxResultSize = listOptions.PageSize + } + if listOptions.NextPageToken != nil { + result.NextPageToken = listOptions.NextPageToken + } + if listOptions.OrderBy != nil { + so := listOptions.SortOrder + + // default is DESC + isAsc := false + if so != nil && *so == "ASC" { + isAsc = true + } + + var orderByField proto.ListOperationOptions_OrderByField_Field + if val, ok := proto.ListOperationOptions_OrderByField_Field_value[*listOptions.OrderBy]; ok { + orderByField = proto.ListOperationOptions_OrderByField_Field(val) + } + + result.OrderByField = &proto.ListOperationOptions_OrderByField{ + Field: &orderByField, + IsAsc: &isAsc, + } + } + return &result, nil +} diff --git a/internal/core/core.go b/internal/core/core.go new file mode 100644 index 00000000..e0cf9752 --- /dev/null +++ b/internal/core/core.go @@ -0,0 +1,456 @@ +package core + +import ( + "context" + "fmt" + "log" + "strconv" + + "github.com/opendatahub-io/model-registry/internal/core/mapper" + "github.com/opendatahub-io/model-registry/internal/ml_metadata/proto" + "github.com/opendatahub-io/model-registry/internal/model/openapi" + "google.golang.org/grpc" +) + +var ( + RegisteredModelTypeName = "odh.RegisteredModel" + ModelVersionTypeName = "odh.ModelVersion" + ModelArtifactTypeName = "odh.ModelArtifact" +) + +// modelRegistryService is the core library of the model registry +type modelRegistryService struct { + mlmdClient proto.MetadataStoreServiceClient + mapper *mapper.Mapper +} + +// NewModelRegistryService create a fresh instance of ModelRegistryService, taking care of setting up needed MLMD Types +func NewModelRegistryService(cc grpc.ClientConnInterface) (ModelRegistryApi, error) { + + client := proto.NewMetadataStoreServiceClient(cc) + + // Setup the needed Type instances if not existing already + + registeredModelReq := proto.PutContextTypeRequest{ + ContextType: &proto.ContextType{ + Name: &RegisteredModelTypeName, + }, + } + + modelVersionReq := proto.PutContextTypeRequest{ + ContextType: &proto.ContextType{ + Name: &ModelVersionTypeName, + Properties: map[string]proto.PropertyType{ + "model_name": proto.PropertyType_STRING, + "version": proto.PropertyType_STRING, + "author": proto.PropertyType_STRING, + }, + }, + } + + modelArtifactReq := proto.PutArtifactTypeRequest{ + ArtifactType: &proto.ArtifactType{ + Name: &ModelArtifactTypeName, + Properties: map[string]proto.PropertyType{ + "model_format": proto.PropertyType_STRING, + }, + }, + } + + registeredModelResp, err := client.PutContextType(context.Background(), ®isteredModelReq) + if err != nil { + log.Fatalf("Error setting up context type %s: %v", RegisteredModelTypeName, err) + } + + modelVersionResp, err := client.PutContextType(context.Background(), &modelVersionReq) + if err != nil { + log.Fatalf("Error setting up context type %s: %v", ModelVersionTypeName, err) + } + modelArtifactResp, err := client.PutArtifactType(context.Background(), &modelArtifactReq) + if err != nil { + log.Fatalf("Error setting up artifact type %s: %v", ModelArtifactTypeName, err) + } + + return &modelRegistryService{ + mlmdClient: client, + mapper: mapper.NewMapper(registeredModelResp.GetTypeId(), modelVersionResp.GetTypeId(), modelArtifactResp.GetTypeId()), + }, nil +} + +// REGISTERED MODELS + +func (serv *modelRegistryService) UpsertRegisteredModel(registeredModel *openapi.RegisteredModel) (*openapi.RegisteredModel, error) { + log.Printf("Creating or updating registered model for %s", *registeredModel.Name) + + modelCtx, err := serv.mapper.MapFromRegisteredModel(registeredModel) + if err != nil { + return nil, err + } + + modelCtxResp, err := serv.mlmdClient.PutContexts(context.Background(), &proto.PutContextsRequest{ + Contexts: []*proto.Context{ + modelCtx, + }, + }) + if err != nil { + return nil, err + } + + modelId := &modelCtxResp.ContextIds[0] + model, err := serv.GetRegisteredModelById((*BaseResourceId)(modelId)) + if err != nil { + return nil, err + } + + return model, nil +} + +func (serv *modelRegistryService) GetRegisteredModelById(id *BaseResourceId) (*openapi.RegisteredModel, error) { + log.Printf("Getting registered model %d", *id) + + getByIdResp, err := serv.mlmdClient.GetContextsByID(context.Background(), &proto.GetContextsByIDRequest{ + ContextIds: []int64{int64(*id)}, + }) + if err != nil { + return nil, err + } + + if len(getByIdResp.Contexts) != 1 { + return nil, fmt.Errorf("multiple registered models found for id %d", *id) + } + + regModel, err := serv.mapper.MapToRegisteredModel(getByIdResp.Contexts[0]) + if err != nil { + return nil, err + } + + return regModel, nil +} + +func (serv *modelRegistryService) GetRegisteredModelByParams(name *string, externalId *string) (*openapi.RegisteredModel, error) { + log.Printf("Getting registered model by params name=%v, externalId=%v", name, externalId) + + filterQuery := "" + if name != nil { + filterQuery = fmt.Sprintf("name = \"%s\"", *name) + } else if externalId != nil { + filterQuery = fmt.Sprintf("external_id = \"%s\"", *externalId) + } + + getByParamsResp, err := serv.mlmdClient.GetContextsByType(context.Background(), &proto.GetContextsByTypeRequest{ + TypeName: &RegisteredModelTypeName, + Options: &proto.ListOperationOptions{ + FilterQuery: &filterQuery, + }, + }) + if err != nil { + return nil, err + } + + if len(getByParamsResp.Contexts) != 1 { + return nil, fmt.Errorf("multiple registered models found for name=%v, externalId=%v", *name, *externalId) + } + + regModel, err := serv.mapper.MapToRegisteredModel(getByParamsResp.Contexts[0]) + if err != nil { + return nil, err + } + return regModel, nil +} + +func (serv *modelRegistryService) GetRegisteredModels(listOptions ListOptions) (*openapi.RegisteredModelList, error) { + listOperationOptions, err := BuildListOperationOptions(listOptions) + if err != nil { + return nil, err + } + contextsResp, err := serv.mlmdClient.GetContextsByType(context.Background(), &proto.GetContextsByTypeRequest{ + TypeName: &RegisteredModelTypeName, + Options: listOperationOptions, + }) + if err != nil { + return nil, err + } + + results := []openapi.RegisteredModel{} + for _, c := range contextsResp.Contexts { + mapped, err := serv.mapper.MapToRegisteredModel(c) + if err != nil { + return nil, err + } + results = append(results, *mapped) + } + + toReturn := openapi.RegisteredModelList{ + NextPageToken: zeroIfNil(contextsResp.NextPageToken), + PageSize: zeroIfNil(listOptions.PageSize), + Size: int32(len(results)), + Items: results, + } + return &toReturn, nil +} + +// MODEL VERSIONS + +func (serv *modelRegistryService) UpsertModelVersion(modelVersion *openapi.ModelVersion, registeredModelId *BaseResourceId) (*openapi.ModelVersion, error) { + registeredModel, err := serv.GetRegisteredModelById(registeredModelId) + if err != nil { + return nil, fmt.Errorf("not a valid registered model id: %d", *registeredModelId) + } + registeredModelIdCtxID, err := mapper.IdToInt64(*registeredModel.Id) + if err != nil { + return nil, err + } + registeredModelName := registeredModel.Name + modelCtx, err := serv.mapper.MapFromModelVersion(modelVersion, *registeredModelIdCtxID, registeredModelName) + if err != nil { + return nil, err + } + + modelCtxResp, err := serv.mlmdClient.PutContexts(context.Background(), &proto.PutContextsRequest{ + Contexts: []*proto.Context{ + modelCtx, + }, + }) + if err != nil { + return nil, err + } + + modelId := &modelCtxResp.ContextIds[0] + _, err = serv.mlmdClient.PutParentContexts(context.Background(), &proto.PutParentContextsRequest{ + ParentContexts: []*proto.ParentContext{{ + ChildId: modelId, + ParentId: registeredModelIdCtxID}}, + TransactionOptions: &proto.TransactionOptions{}, + }) + if err != nil { + return nil, err + } + + model, err := serv.GetModelVersionById((*BaseResourceId)(modelId)) + if err != nil { + return nil, err + } + + return model, nil +} + +func (serv *modelRegistryService) GetModelVersionById(id *BaseResourceId) (*openapi.ModelVersion, error) { + getByIdResp, err := serv.mlmdClient.GetContextsByID(context.Background(), &proto.GetContextsByIDRequest{ + ContextIds: []int64{int64(*id)}, + }) + if err != nil { + return nil, err + } + + if len(getByIdResp.Contexts) != 1 { + return nil, fmt.Errorf("multiple model versions found for id %d", *id) + } + + modelVer, err := serv.mapper.MapToModelVersion(getByIdResp.Contexts[0]) + if err != nil { + return nil, err + } + + return modelVer, nil +} + +func (serv *modelRegistryService) GetModelVersionByParams(name *string, externalId *string) (*openapi.ModelVersion, error) { + filterQuery := "" + if name != nil { + filterQuery = fmt.Sprintf("name = \"%s\"", *name) + } else if externalId != nil { + filterQuery = fmt.Sprintf("external_id = \"%s\"", *externalId) + } + + getByParamsResp, err := serv.mlmdClient.GetContextsByType(context.Background(), &proto.GetContextsByTypeRequest{ + TypeName: &ModelVersionTypeName, + Options: &proto.ListOperationOptions{ + FilterQuery: &filterQuery, + }, + }) + if err != nil { + return nil, err + } + + if len(getByParamsResp.Contexts) != 1 { + return nil, fmt.Errorf("multiple registered models found for name=%v, externalId=%v", *name, *externalId) + } + + modelVer, err := serv.mapper.MapToModelVersion(getByParamsResp.Contexts[0]) + if err != nil { + return nil, err + } + return modelVer, nil +} + +func (serv *modelRegistryService) GetModelVersions(listOptions ListOptions, registeredModelId *BaseResourceId) (*openapi.ModelVersionList, error) { + listOperationOptions, err := BuildListOperationOptions(listOptions) + if err != nil { + return nil, err + } + + if registeredModelId != nil { + queryParentCtxId := fmt.Sprintf("parent_contexts_a.type = %d", *registeredModelId) + listOperationOptions.FilterQuery = &queryParentCtxId + } + + contextsResp, err := serv.mlmdClient.GetContextsByType(context.Background(), &proto.GetContextsByTypeRequest{ + TypeName: &ModelVersionTypeName, + Options: listOperationOptions, + }) + if err != nil { + return nil, err + } + + results := []openapi.ModelVersion{} + for _, c := range contextsResp.Contexts { + mapped, err := serv.mapper.MapToModelVersion(c) + if err != nil { + return nil, err + } + results = append(results, *mapped) + } + + toReturn := openapi.ModelVersionList{ + NextPageToken: zeroIfNil(contextsResp.NextPageToken), + PageSize: zeroIfNil(listOptions.PageSize), + Size: int32(len(results)), + Items: results, + } + return &toReturn, nil +} + +// MODEL ARTIFACTS + +func (serv *modelRegistryService) UpsertModelArtifact(modelArtifact *openapi.ModelArtifact, modelVersionId *BaseResourceId) (*openapi.ModelArtifact, error) { + artifact := serv.mapper.MapFromModelArtifact(*modelArtifact) + + artifactsResp, err := serv.mlmdClient.PutArtifacts(context.Background(), &proto.PutArtifactsRequest{ + Artifacts: []*proto.Artifact{artifact}, + }) + if err != nil { + return nil, err + } + idString := strconv.FormatInt(artifactsResp.ArtifactIds[0], 10) + modelArtifact.Id = &idString + + // add explicit association between artifacts and model version + if modelVersionId != nil { + modelVersionIdCtx := int64(*modelVersionId) + attributions := []*proto.Attribution{} + for _, a := range artifactsResp.ArtifactIds { + attributions = append(attributions, &proto.Attribution{ + ContextId: &modelVersionIdCtx, + ArtifactId: &a, + }) + } + _, err = serv.mlmdClient.PutAttributionsAndAssociations(context.Background(), &proto.PutAttributionsAndAssociationsRequest{ + Attributions: attributions, + Associations: make([]*proto.Association, 0), + }) + if err != nil { + return nil, err + } + } + + return modelArtifact, nil +} + +func (serv *modelRegistryService) GetModelArtifactById(id *BaseResourceId) (*openapi.ModelArtifact, error) { + artifactsResp, err := serv.mlmdClient.GetArtifactsByID(context.Background(), &proto.GetArtifactsByIDRequest{ + ArtifactIds: []int64{int64(*id)}, + }) + if err != nil { + return nil, err + } + + result, err := serv.mapper.MapToModelArtifact(artifactsResp.Artifacts[0]) + if err != nil { + return nil, err + } + + return result, nil +} + +func (serv *modelRegistryService) GetModelArtifactByParams(name *string, externalId *string) (*openapi.ModelArtifact, error) { + var artifact0 *proto.Artifact + + filterQuery := "" + if externalId != nil { + filterQuery = fmt.Sprintf("external_id = \"%s\"", *externalId) + } else if name != nil { + filterQuery = fmt.Sprintf("name = \"%s\"", *name) + } else { + return nil, fmt.Errorf("invalid parameters call, supply either name or externalId") + } + + artifactsResponse, err := serv.mlmdClient.GetArtifactsByType(context.Background(), &proto.GetArtifactsByTypeRequest{ + TypeName: &ModelArtifactTypeName, + Options: &proto.ListOperationOptions{ + FilterQuery: &filterQuery, + }, + }) + if err != nil { + return nil, err + } + if len(artifactsResponse.Artifacts) > 1 { + return nil, fmt.Errorf("more than an artifact detected matching criteria: %v", artifactsResponse.Artifacts) + } + artifact0 = artifactsResponse.Artifacts[0] + + result, err := serv.mapper.MapToModelArtifact(artifact0) + if err != nil { + return nil, err + } + + return result, nil +} + +func (serv *modelRegistryService) GetModelArtifacts(listOptions ListOptions, modelVersionId *BaseResourceId) (*openapi.ModelArtifactList, error) { + listOperationOptions, err := BuildListOperationOptions(listOptions) + if err != nil { + return nil, err + } + + var artifacts []*proto.Artifact + var nextPageToken *string + if modelVersionId != nil { + ctxId := int64(*modelVersionId) + artifactsResp, err := serv.mlmdClient.GetArtifactsByContext(context.Background(), &proto.GetArtifactsByContextRequest{ + ContextId: &ctxId, + Options: listOperationOptions, + }) + if err != nil { + return nil, err + } + artifacts = artifactsResp.Artifacts + nextPageToken = artifactsResp.NextPageToken + } else { + artifactsResp, err := serv.mlmdClient.GetArtifactsByType(context.Background(), &proto.GetArtifactsByTypeRequest{ + TypeName: &ModelArtifactTypeName, + Options: listOperationOptions, + }) + if err != nil { + return nil, err + } + artifacts = artifactsResp.Artifacts + nextPageToken = artifactsResp.NextPageToken + } + + results := []openapi.ModelArtifact{} + for _, a := range artifacts { + mapped, err := serv.mapper.MapToModelArtifact(a) + if err != nil { + return nil, err + } + results = append(results, *mapped) + } + + toReturn := openapi.ModelArtifactList{ + NextPageToken: zeroIfNil(nextPageToken), + PageSize: zeroIfNil(listOptions.PageSize), + Size: int32(len(results)), + Items: results, + } + return &toReturn, nil +} diff --git a/internal/core/core_test.go b/internal/core/core_test.go new file mode 100644 index 00000000..dfb5c3d1 --- /dev/null +++ b/internal/core/core_test.go @@ -0,0 +1,198 @@ +package core_test + +import ( + "context" + "fmt" + "os" + "testing" + + "github.com/opendatahub-io/model-registry/internal/core" + "github.com/opendatahub-io/model-registry/internal/core/mapper" + "github.com/opendatahub-io/model-registry/internal/ml_metadata/proto" + "github.com/opendatahub-io/model-registry/internal/model/openapi" + "github.com/stretchr/testify/assert" + "github.com/testcontainers/testcontainers-go" + "github.com/testcontainers/testcontainers-go/wait" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" +) + +const ( + useProvider = testcontainers.ProviderDefault // or explicit to testcontainers.ProviderPodman if needed + mlmdImage = "gcr.io/tfx-oss-public/ml_metadata_store_server:1.14.0" + sqliteFile = "metadata.sqlite.db" + testConfigFolder = "test/config/ml-metadata" +) + +func TestCreateRegisteredModel(t *testing.T) { + conn, client, teardown := SetupTestContainer(t) + defer teardown(t) + + // [TEST CASE] + + // create mode registry service + service, err := core.NewModelRegistryService(conn) + assert.Nilf(t, err, "error creating core service: %v", err) + + modelName := "PricingModel" + externalId := "myExternalId" + owner := "Myself" + + // register a new model + registeredModel := &openapi.RegisteredModel{ + Name: &modelName, + ExternalID: &externalId, + CustomProperties: &map[string]openapi.MetadataValue{ + "owner": { + MetadataValueOneOf2: &openapi.MetadataValueOneOf2{ + StringValue: &owner, + }, + }, + }, + } + + // test + createdModel, err := service.UpsertRegisteredModel(registeredModel) + + // checks + assert.Nilf(t, err, "error creating registered model: %v", err) + assert.NotNilf(t, createdModel.Id, "created registered model should not have nil Id") + + byTypeAndNameResp, err := client.GetContextByTypeAndName(context.Background(), &proto.GetContextByTypeAndNameRequest{ + TypeName: &core.RegisteredModelTypeName, + ContextName: &modelName, + }) + assert.Nilf(t, err, "error retrieving context by type and name, not related to the test itself: %v", err) + + ctxId := mapper.IdToString(*byTypeAndNameResp.Context.Id) + assert.Equal(t, *createdModel.Id, *ctxId, "returned model id should match the mlmd one") + assert.Equal(t, modelName, *byTypeAndNameResp.Context.Name, "saved model name should match the provided one") + assert.Equal(t, externalId, *byTypeAndNameResp.Context.ExternalId, "saved external id should match the provided one") + assert.Equal(t, owner, byTypeAndNameResp.Context.CustomProperties["owner"].GetStringValue(), "saved owner custom property should match the provided one") + + getAllResp, err := client.GetContexts(context.Background(), &proto.GetContextsRequest{}) + assert.Nilf(t, err, "error retrieving all contexts, not related to the test itself: %v", err) + assert.Equal(t, 1, len(getAllResp.Contexts), "there should be just one context saved in mlmd") +} + +func TestGetRegisteredModelById(t *testing.T) { + conn, _, teardown := SetupTestContainer(t) + defer teardown(t) + + // [TEST CASE] + + // create mode registry service + service, err := core.NewModelRegistryService(conn) + assert.Nilf(t, err, "error creating core service: %v", err) + + modelName := "PricingModel" + externalId := "mysupermodel" + + // register a new model + registeredModel := &openapi.RegisteredModel{ + Name: &modelName, + ExternalID: &externalId, + } + + // test + createdModel, err := service.UpsertRegisteredModel(registeredModel) + + // checks + assert.Nilf(t, err, "error creating registered model: %v", err) + + modelId, _ := mapper.IdToInt64(*createdModel.Id) + getModelById, err := service.GetRegisteredModelById((*core.BaseResourceId)(modelId)) + assert.Nilf(t, err, "error getting registered model by id %d: %v", *modelId, err) + + assert.Equal(t, modelName, *getModelById.Name, "saved model name should match the provided one") + assert.Equal(t, externalId, *getModelById.ExternalID, "saved external id should match the provided one") +} + +// ################# +// ##### Utils ##### +// ################# + +func clearMetadataSqliteDB(wd string) error { + if err := os.Remove(fmt.Sprintf("%s/%s", wd, sqliteFile)); err != nil { + return fmt.Errorf("expected to clear sqlite file but didn't find: %v", err) + } + return nil +} + +// SetupTestContainer creates a MLMD gRPC test container +// Returns +// - gRPC client connection to the test container +// - ml-metadata client used to double check the database +// - teardown function +func SetupTestContainer(t *testing.T) (*grpc.ClientConn, proto.MetadataStoreServiceClient, func(t *testing.T)) { + ctx := context.Background() + wd, err := os.Getwd() + if err != nil { + t.Errorf("error getting working directory: %v", err) + } + wd = fmt.Sprintf("%s/../../%s", wd, testConfigFolder) + t.Logf("using working directory: %s", wd) + + req := testcontainers.ContainerRequest{ + Image: mlmdImage, + ExposedPorts: []string{"8080/tcp"}, + Env: map[string]string{ + "METADATA_STORE_SERVER_CONFIG_FILE": "/tmp/shared/conn_config.pb", + }, + Mounts: testcontainers.ContainerMounts{ + testcontainers.ContainerMount{ + Source: testcontainers.GenericBindMountSource{ + HostPath: wd, + }, + Target: "/tmp/shared", + }, + }, + WaitingFor: wait.ForLog("Server listening on"), + } + + mlmdgrpc, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ + ProviderType: useProvider, + ContainerRequest: req, + Started: true, + }) + if err != nil { + t.Errorf("error setting up mlmd grpc container: %v", err) + } + + mappedHost, err := mlmdgrpc.Host(ctx) + if err != nil { + t.Error(err) + } + mappedPort, err := mlmdgrpc.MappedPort(ctx, "8080") + if err != nil { + t.Error(err) + } + mlmdAddr := fmt.Sprintf("%s:%s", mappedHost, mappedPort.Port()) + t.Log("MLMD test container setup at: ", mlmdAddr) + + // setup grpc connection + conn, err := grpc.DialContext( + context.Background(), + mlmdAddr, + grpc.WithReturnConnectionError(), + grpc.WithBlock(), + grpc.WithTransportCredentials(insecure.NewCredentials()), + ) + if err != nil { + t.Errorf("error dialing connection to mlmd server %s: %v", mlmdAddr, err) + } + + mlmdClient := proto.NewMetadataStoreServiceClient(conn) + + return conn, mlmdClient, func(t *testing.T) { + if err := conn.Close(); err != nil { + t.Error(err) + } + if err := mlmdgrpc.Terminate(ctx); err != nil { + t.Error(err) + } + if err := clearMetadataSqliteDB(wd); err != nil { + t.Error(err) + } + } +} diff --git a/internal/core/mapper/mlmd_mapper.go b/internal/core/mapper/mlmd_mapper.go new file mode 100644 index 00000000..a657d0de --- /dev/null +++ b/internal/core/mapper/mlmd_mapper.go @@ -0,0 +1,263 @@ +package mapper + +import ( + "fmt" + "log" + "strconv" + + "github.com/opendatahub-io/model-registry/internal/ml_metadata/proto" + "github.com/opendatahub-io/model-registry/internal/model/openapi" +) + +type Mapper struct { + RegisteredModelTypeId int64 + ModelVersionTypeId int64 + ModelArtifactTypeId int64 +} + +func NewMapper(registeredModelTypeId int64, modelVersionTypeId int64, modelArtifactTypeId int64) *Mapper { + return &Mapper{ + RegisteredModelTypeId: registeredModelTypeId, + ModelVersionTypeId: modelVersionTypeId, + ModelArtifactTypeId: modelArtifactTypeId, + } +} + +func IdToInt64(idString string) (*int64, error) { + idInt, err := strconv.Atoi(idString) + if err != nil { + return nil, err + } + + idInt64 := int64(idInt) + + return &idInt64, nil +} + +func IdToString(idInt int64) *string { + idString := strconv.FormatInt(idInt, 10) + + return &idString +} + +// Internal Model --> MLMD + +// Map generic map into MLMD [custom] properties object +func (m *Mapper) MapToProperties(data map[string]openapi.MetadataValue) (map[string]*proto.Value, error) { + props := make(map[string]*proto.Value) + + for key, v := range data { + value := proto.Value{} + + switch { + // int value + case v.MetadataValueOneOf != nil: + intValue, err := IdToInt64(*v.MetadataValueOneOf.IntValue) + if err != nil { + log.Printf("Skipping mapping for %s:%v", key, v) + continue + } + value.Value = &proto.Value_IntValue{IntValue: *intValue} + // double value + case v.MetadataValueOneOf1 != nil: + value.Value = &proto.Value_DoubleValue{DoubleValue: *v.MetadataValueOneOf1.DoubleValue} + // double value + case v.MetadataValueOneOf2 != nil: + value.Value = &proto.Value_StringValue{StringValue: *v.MetadataValueOneOf2.StringValue} + default: + log.Printf("Type mapping not found for %s:%v", key, v) + continue + } + + props[key] = &value + } + + return props, nil +} + +func (m *Mapper) MapFromRegisteredModel(registeredModel *openapi.RegisteredModel) (*proto.Context, error) { + + var idInt *int64 + if registeredModel.Id != nil { + var err error + idInt, err = IdToInt64(*registeredModel.Id) + if err != nil { + return nil, err + } + } + + customProps := make(map[string]*proto.Value) + if registeredModel.CustomProperties != nil { + customProps, _ = m.MapToProperties(*registeredModel.CustomProperties) + } + + return &proto.Context{ + Name: registeredModel.Name, + TypeId: &m.RegisteredModelTypeId, + ExternalId: registeredModel.ExternalID, + Id: idInt, + CustomProperties: customProps, + }, nil +} + +func (m *Mapper) MapFromModelVersion(modelVersion *openapi.ModelVersion, registeredModelId int64, registeredModelName *string) (*proto.Context, error) { + fullName := fmt.Sprintf("%d:%s", registeredModelId, *modelVersion.Name) + customProps := make(map[string]*proto.Value) + if modelVersion.CustomProperties != nil { + customProps, _ = m.MapToProperties(*modelVersion.CustomProperties) + } + ctx := &proto.Context{ + Name: &fullName, + TypeId: &m.ModelVersionTypeId, + Properties: map[string]*proto.Value{ + "model_name": { + Value: &proto.Value_StringValue{ + StringValue: *registeredModelName, + }, + }, + }, + CustomProperties: customProps, + } + if modelVersion.Name != nil { + ctx.Properties["version"] = &proto.Value{ + Value: &proto.Value_StringValue{ + StringValue: *modelVersion.Name, + }, + } + } + // TODO: missing explicit property in openapi + // if modelVersion.Author != nil { + // ctx.Properties["author"] = &proto.Value{ + // Value: &proto.Value_StringValue{ + // StringValue: *modelVersion.Author, + // }, + // } + // } + + return ctx, nil +} + +func (m *Mapper) MapFromModelArtifact(modelArtifact openapi.ModelArtifact) *proto.Artifact { + return &proto.Artifact{ + TypeId: &m.ModelArtifactTypeId, + // TODO: we should use concatenation between uuid + name + Name: modelArtifact.Name, + Uri: modelArtifact.Uri, + } +} + +func (m *Mapper) MapFromModelArtifacts(modelArtifacts *[]openapi.ModelArtifact) ([]*proto.Artifact, error) { + artifacts := []*proto.Artifact{} + if modelArtifacts == nil { + return artifacts, nil + } + for _, a := range *modelArtifacts { + artifacts = append(artifacts, m.MapFromModelArtifact(a)) + } + return artifacts, nil +} + +// MLMD --> Internal Model + +// Maps MLMD properties into a generic map +func (m *Mapper) MapFromProperties(props map[string]*proto.Value) (map[string]openapi.MetadataValue, error) { + data := make(map[string]openapi.MetadataValue) + + for key, v := range props { + // data[key] = v.Value + customValue := openapi.MetadataValue{} + + switch typedValue := v.Value.(type) { + case *proto.Value_IntValue: + customValue.MetadataValueOneOf = &openapi.MetadataValueOneOf{ + IntValue: IdToString(typedValue.IntValue), + } + case *proto.Value_DoubleValue: + customValue.MetadataValueOneOf1 = &openapi.MetadataValueOneOf1{ + DoubleValue: &typedValue.DoubleValue, + } + case *proto.Value_StringValue: + customValue.MetadataValueOneOf2 = &openapi.MetadataValueOneOf2{ + StringValue: &typedValue.StringValue, + } + default: + log.Printf("Type mapping not found for %s:%v", key, v) + continue + } + + data[key] = customValue + } + + return data, nil +} + +func (m *Mapper) MapToRegisteredModel(ctx *proto.Context) (*openapi.RegisteredModel, error) { + if ctx.GetTypeId() != m.RegisteredModelTypeId { + return nil, fmt.Errorf("invalid TypeId, exptected %d but received %d", m.RegisteredModelTypeId, ctx.GetTypeId()) + } + + _, err := m.MapFromProperties(ctx.CustomProperties) + if err != nil { + return nil, err + } + + idString := strconv.FormatInt(*ctx.Id, 10) + + model := &openapi.RegisteredModel{ + Id: &idString, + Name: ctx.Name, + ExternalID: ctx.ExternalId, + } + + return model, nil +} + +func (m *Mapper) MapToModelVersion(ctx *proto.Context) (*openapi.ModelVersion, error) { + if ctx.GetTypeId() != m.ModelVersionTypeId { + return nil, fmt.Errorf("invalid TypeId, exptected %d but received %d", m.ModelVersionTypeId, ctx.GetTypeId()) + } + + metadata, err := m.MapFromProperties(ctx.CustomProperties) + if err != nil { + return nil, err + } + + // modelName := ctx.GetProperties()["model_name"].GetStringValue() + // version := ctx.GetProperties()["version"].GetStringValue() + // author := ctx.GetProperties()["author"].GetStringValue() + + idString := strconv.FormatInt(*ctx.Id, 10) + + modelVersion := &openapi.ModelVersion{ + // ModelName: &modelName, + Id: &idString, + Name: ctx.Name, + // Author: &author, + CustomProperties: &metadata, + } + + return modelVersion, nil +} + +func (m *Mapper) MapToModelArtifact(artifact *proto.Artifact) (*openapi.ModelArtifact, error) { + if artifact.GetTypeId() != m.ModelArtifactTypeId { + return nil, fmt.Errorf("invalid TypeId, exptected %d but received %d", m.ModelArtifactTypeId, artifact.GetTypeId()) + } + + _, err := m.MapFromProperties(artifact.CustomProperties) + if err != nil { + return nil, err + } + + _, err = m.MapFromProperties(artifact.Properties) + if err != nil { + return nil, err + } + + modelArtifact := &openapi.ModelArtifact{ + Uri: artifact.Uri, + Name: artifact.Name, + } + + return modelArtifact, nil +} diff --git a/test/bdd/conn_config.pb b/test/bdd/conn_config.pb new file mode 100644 index 00000000..774b7aa0 --- /dev/null +++ b/test/bdd/conn_config.pb @@ -0,0 +1,6 @@ +connection_config { + sqlite { + filename_uri: '/tmp/shared/metadata.sqlite.db' + connection_mode: READWRITE_OPENCREATE + } +} diff --git a/test/bdd/features/example.feature b/test/bdd/features/example.feature new file mode 100644 index 00000000..70049726 --- /dev/null +++ b/test/bdd/features/example.feature @@ -0,0 +1,9 @@ +Feature: As a MLOps engineer I would like the Model Registry to store metadata information about models + Taken from User Stories + + Scenario: As a MLOps engineer I would like to store Model name + Given I have a connection to MR + When I store a RegisteredModel with name "Pricing Model" and a child ModelVersion with name "v1" and a child Artifact with uri "s3://12345" + Then there should be a mlmd Context of type "odh.RegisteredModel" named "Pricing Model" + And there should be a mlmd Context of type "odh.ModelVersion" having property named "model_name" valorised with string value "Pricing Model" + diff --git a/test/bdd/mr_service_features_test.go b/test/bdd/mr_service_features_test.go new file mode 100644 index 00000000..5def7f11 --- /dev/null +++ b/test/bdd/mr_service_features_test.go @@ -0,0 +1,228 @@ +package bdd_test + +import ( + "context" + "fmt" + "log" + "os" + "strconv" + "testing" + + "github.com/cucumber/godog" + "github.com/opendatahub-io/model-registry/internal/core" + "github.com/opendatahub-io/model-registry/internal/ml_metadata/proto" + "github.com/opendatahub-io/model-registry/internal/model/openapi" + "github.com/testcontainers/testcontainers-go" + "github.com/testcontainers/testcontainers-go/wait" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" +) + +const useProvider = testcontainers.ProviderDefault // or explicit to testcontainers.ProviderPodman if needed + +var ( + mlmdHostname string + mlmdPort int +) + +func TestFeatures(t *testing.T) { + suite := godog.TestSuite{ + ScenarioInitializer: InitializeScenario, + Options: &godog.Options{ + Format: "pretty", + Paths: []string{"features"}, + TestingT: t, // Testing instance that will run subtests. + }, + } + + if suite.Run() != 0 { + t.Fatal("non-zero status returned, failed to run feature tests") + } +} + +type wdCtxKey struct{} +type testContainerCtxKey struct{} +type svcLayerCtxKey struct{} +type connCtxKey struct{} + +func iHaveAConnectionToMR(ctx context.Context) (context.Context, error) { + mlmdAddr := fmt.Sprintf("%s:%d", mlmdHostname, mlmdPort) + conn, err := grpc.DialContext( + context.Background(), + mlmdAddr, + grpc.WithReturnConnectionError(), + grpc.WithBlock(), + grpc.WithTransportCredentials(insecure.NewCredentials()), + ) + if err != nil { + log.Fatalf("Error dialing connection to mlmd server %s: %v", mlmdAddr, err) + return nil, err + } + ctx = context.WithValue(ctx, connCtxKey{}, conn) + service, err := core.NewModelRegistryService(conn) + if err != nil { + log.Fatalf("Error creating core service: %v", err) + return nil, err + } + return context.WithValue(ctx, svcLayerCtxKey{}, service), nil +} + +func iStoreARegisteredModelWithNameAndAChildModelVersionWithNameAndAChildArtifactWithUri(ctx context.Context, registedModelName, modelVersionName, artifactURI string) error { + service, ok := ctx.Value(svcLayerCtxKey{}).(core.ModelRegistryApi) + if !ok { + return fmt.Errorf("not found service layer connection in godog context") + } + var registeredModel *openapi.RegisteredModel + var err error + registeredModel, err = service.UpsertRegisteredModel(&openapi.RegisteredModel{ + Name: ®istedModelName, + }) + if err != nil { + return err + } + registeredModelId, err := idToInt64(*registeredModel.Id) + if err != nil { + return err + } + + var modelVersion *openapi.ModelVersion + if modelVersion, err = service.UpsertModelVersion(&openapi.ModelVersion{Name: &modelVersionName}, (*core.BaseResourceId)(registeredModelId)); err != nil { + return err + } + modelVersionId, err := idToInt64(*modelVersion.Id) + if err != nil { + return err + } + + if _, err = service.UpsertModelArtifact(&openapi.ModelArtifact{Uri: &artifactURI}, (*core.BaseResourceId)(modelVersionId)); err != nil { + return err + } + + return nil +} + +func idToInt64(idString string) (*int64, error) { + idInt, err := strconv.Atoi(idString) + if err != nil { + return nil, err + } + + idInt64 := int64(idInt) + + return &idInt64, nil +} + +func thereShouldBeAMlmdContextOfTypeNamed(ctx context.Context, arg1, arg2 string) error { + conn := ctx.Value(connCtxKey{}).(*grpc.ClientConn) + client := proto.NewMetadataStoreServiceClient(conn) + query := fmt.Sprintf("type = \"%s\" and name = \"%s\"", arg1, arg2) + fmt.Println("query: ", query) + resp, err := client.GetContexts(context.Background(), &proto.GetContextsRequest{ + Options: &proto.ListOperationOptions{ + FilterQuery: &query, + }, + }) + if err != nil { + return err + } + if len(resp.Contexts) != 1 { + return fmt.Errorf("Unexpected mlmd Context result size (%d), %v", len(resp.Contexts), resp.Contexts) + } + return nil +} + +func thereShouldBeAMlmdContextOfTypeHavingPropertyNamedValorisedWithStringValue(ctx context.Context, arg1, arg2, arg3 string) error { + conn := ctx.Value(connCtxKey{}).(*grpc.ClientConn) + client := proto.NewMetadataStoreServiceClient(conn) + query := fmt.Sprintf("type = \"%s\" and properties.%s.string_value = \"%s\"", arg1, arg2, arg3) + fmt.Println("query: ", query) + resp, err := client.GetContexts(context.Background(), &proto.GetContextsRequest{ + Options: &proto.ListOperationOptions{ + FilterQuery: &query, + }, + }) + if err != nil { + return err + } + if len(resp.Contexts) != 1 { + return fmt.Errorf("Unexpected mlmd Context result size (%d), %v", len(resp.Contexts), resp.Contexts) + } + return nil +} + +func InitializeScenario(ctx *godog.ScenarioContext) { + ctx.Before(func(ctx context.Context, sc *godog.Scenario) (context.Context, error) { + wd, mlmdgrpc, err := setupTestContainer(ctx) + if err != nil { + return ctx, err + } + ctx = context.WithValue(ctx, wdCtxKey{}, wd) + ctx = context.WithValue(ctx, testContainerCtxKey{}, mlmdgrpc) + mappedHost, err := mlmdgrpc.Host(ctx) + if err != nil { + return ctx, err + } + mappedPort, err := mlmdgrpc.MappedPort(ctx, "8080") + if err != nil { + return ctx, err + } + // TODO: these are effectively global in main and could be worthy to revisit + mlmdHostname = mappedHost + mlmdPort = mappedPort.Int() + return ctx, nil + }) + ctx.Step(`^I have a connection to MR$`, iHaveAConnectionToMR) + ctx.Step(`^I store a RegisteredModel with name "([^"]*)" and a child ModelVersion with name "([^"]*)" and a child Artifact with uri "([^"]*)"$`, iStoreARegisteredModelWithNameAndAChildModelVersionWithNameAndAChildArtifactWithUri) + ctx.Step(`^there should be a mlmd Context of type "([^"]*)" named "([^"]*)"$`, thereShouldBeAMlmdContextOfTypeNamed) + ctx.Step(`^there should be a mlmd Context of type "([^"]*)" having property named "([^"]*)" valorised with string value "([^"]*)"$`, thereShouldBeAMlmdContextOfTypeHavingPropertyNamedValorisedWithStringValue) + ctx.After(func(ctx context.Context, sc *godog.Scenario, err error) (context.Context, error) { + conn := ctx.Value(connCtxKey{}).(*grpc.ClientConn) + conn.Close() + mlmdgrpc := ctx.Value(testContainerCtxKey{}).(testcontainers.Container) + if err := mlmdgrpc.Terminate(ctx); err != nil { + return ctx, err + } + wd := ctx.Value(wdCtxKey{}).(string) + clearMetadataSqliteDB(wd) + return ctx, nil + }) +} + +func clearMetadataSqliteDB(wd string) error { + if err := os.Remove(fmt.Sprintf("%s/%s", wd, "metadata.sqlite.db")); err != nil { + return fmt.Errorf("Expected to clear sqlite file but didn't find: %v", err) + } + return nil +} + +func setupTestContainer(ctx context.Context) (string, testcontainers.Container, error) { + wd, err := os.Getwd() + if err != nil { + return "", nil, err + } + req := testcontainers.ContainerRequest{ + Image: "gcr.io/tfx-oss-public/ml_metadata_store_server:1.14.0", + ExposedPorts: []string{"8080/tcp"}, + Env: map[string]string{ + "METADATA_STORE_SERVER_CONFIG_FILE": "/tmp/shared/conn_config.pb", + }, + Mounts: testcontainers.ContainerMounts{ + testcontainers.ContainerMount{ + Source: testcontainers.GenericBindMountSource{ + HostPath: wd, + }, + Target: "/tmp/shared", + }, + }, + WaitingFor: wait.ForLog("Server listening on"), + } + mlmdgrpc, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ + ProviderType: useProvider, + ContainerRequest: req, + Started: true, + }) + if err != nil { + return "", nil, err + } + return wd, mlmdgrpc, nil +} diff --git a/test/config/ml-metadata/conn_config.pb b/test/config/ml-metadata/conn_config.pb new file mode 100644 index 00000000..774b7aa0 --- /dev/null +++ b/test/config/ml-metadata/conn_config.pb @@ -0,0 +1,6 @@ +connection_config { + sqlite { + filename_uri: '/tmp/shared/metadata.sqlite.db' + connection_mode: READWRITE_OPENCREATE + } +}