diff --git a/boilerplate/lyft/golang_support_tools/tools.go b/boilerplate/lyft/golang_support_tools/tools.go index 4310b39d..88ff6452 100644 --- a/boilerplate/lyft/golang_support_tools/tools.go +++ b/boilerplate/lyft/golang_support_tools/tools.go @@ -3,8 +3,8 @@ package tools import ( + _ "github.com/alvaroloes/enumer" _ "github.com/golangci/golangci-lint/cmd/golangci-lint" _ "github.com/lyft/flytestdlib/cli/pflags" _ "github.com/vektra/mockery/cmd/mockery" - _ "github.com/alvaroloes/enumer" ) diff --git a/cmd/entrypoints/migrate.go b/cmd/entrypoints/migrate.go index 79514132..96635381 100644 --- a/cmd/entrypoints/migrate.go +++ b/cmd/entrypoints/migrate.go @@ -9,7 +9,6 @@ import ( "context" - _ "github.com/jinzhu/gorm/dialects/postgres" // Required to import database driver. "github.com/spf13/cobra" ) @@ -73,7 +72,10 @@ var migrateCmd = &cobra.Command{ logger.Infof(ctx, "Created DB connection.") // TODO: checkpoints for migrations - dbHandle.Migrate() + if err := dbHandle.Migrate(ctx); err != nil { + logger.Errorf(ctx, "Failed to migrate. err: %v", err) + panic(err) + } logger.Infof(ctx, "Ran DB migration successfully.") }, } diff --git a/datacatalog_config.yaml b/datacatalog_config.yaml index 19788f01..e0f9c3d1 100644 --- a/datacatalog_config.yaml +++ b/datacatalog_config.yaml @@ -30,3 +30,4 @@ database: host: localhost dbname: datacatalog options: "sslmode=disable" + log_level: 5 diff --git a/go.mod b/go.mod index 920f6ace..346c2e5c 100644 --- a/go.mod +++ b/go.mod @@ -8,12 +8,12 @@ require ( github.com/flyteorg/flytestdlib v0.3.13 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b github.com/golang/protobuf v1.4.3 - github.com/jinzhu/gorm v1.9.11 - github.com/lib/pq v1.2.0 + github.com/lib/pq v1.3.0 github.com/mitchellh/mapstructure v1.4.1 github.com/spf13/cobra v1.1.1 github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.7.0 google.golang.org/grpc v1.36.0 - google.golang.org/protobuf v1.25.0 + gorm.io/driver/postgres v1.1.0 + gorm.io/gorm v1.21.9 ) diff --git a/go.sum b/go.sum index 087a825d..5742d0d6 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,5 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.37.4/go.mod h1:NHPJ89PdicEuT9hdPXMROBD91xc5uRDxsMtSB16k7hw= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= @@ -71,6 +70,8 @@ github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBp github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= +github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc= +github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= @@ -126,6 +127,8 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I= +github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/coocood/freecache v1.1.1 h1:uukNF7QKCZEdZ9gAV7WQzvh0SbjwdMF6m3x3rxEkaPc= @@ -137,6 +140,7 @@ github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= @@ -145,8 +149,6 @@ github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7Do github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/denisenkom/go-mssqldb v0.0.0-20190515213511-eb9f6a1743f3 h1:tkum0XDgfR0jcVVXuTsYv/erY2NnEDqwRojbxR1rBYA= -github.com/denisenkom/go-mssqldb v0.0.0-20190515213511-eb9f6a1743f3/go.mod h1:zAg7JM8CkOJ43xKXIj7eRO9kmWm/TW578qo+oDO6tuM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dnaeon/go-vcr v1.1.0 h1:ReYa/UBrRyQdant9B4fNHGoCNKw6qh6P0fsdGmZpR7c= @@ -167,16 +169,14 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5 h1:Yzb9+7DPaBjB8zlTR87/ElzFsnQfuHnVUVqpZZIcV5Y= -github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5/go.mod h1:a2zkGnVExMxdzMo3M0Hi/3sEU+cWnZpSni0O6/Yb/P0= github.com/ernesto-jimenez/gogen v0.0.0-20180125220232-d7d4131e6607/go.mod h1:Cg4fM0vhYWOZdgM7RIOSTRNIc8/VT7CXClC3Ni86lu4= github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.10.0 h1:s36xzo75JdqLaaWoiEHk767eHiwo0598uUxyfiPkDsg= github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= github.com/fatih/structtag v1.2.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4/aAZl94= -github.com/flyteorg/flyteidl v0.18.15 h1:sXrlwTRaRjQsXYMNrY/S930SKdKtu4XnpNFEu8I4tn4= -github.com/flyteorg/flyteidl v0.18.15/go.mod h1:b5Fq4Z8a5b0mF6pEwTd48ufvikUGVkWSjZiMT0ZtqKI= +github.com/flyteorg/flyteidl v0.18.17 h1:74pPZ9PzITuzq+CgjMPb9EcFI5bVkf8mM5m4xmmlTmY= +github.com/flyteorg/flyteidl v0.18.17/go.mod h1:b5Fq4Z8a5b0mF6pEwTd48ufvikUGVkWSjZiMT0ZtqKI= github.com/flyteorg/flytestdlib v0.3.13 h1:5ioA/q3ixlyqkFh5kDaHgmPyTP/AHtqq1K/TIbVLUzM= github.com/flyteorg/flytestdlib v0.3.13/go.mod h1:Tz8JCECAbX6VWGwFT6cmEQ+RJpZ/6L9pswu3fzWs220= github.com/form3tech-oss/jwt-go v3.2.2+incompatible h1:TcekIExNqud5crz4xD2pavyTgWiPvpYe4Xau31I0PRk= @@ -210,10 +210,10 @@ github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8 github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= -github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA= -github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-test/deep v1.0.7/go.mod h1:QV8Hv/iy04NyLBxAdO9njL0iVPN1S4d/A3NVv1V36o8= +github.com/gofrs/uuid v3.2.0+incompatible h1:y12jRkkFxsd7GpqdSZ+/KCs/fJbqpEXSGd4+jfEaewE= +github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= @@ -341,12 +341,62 @@ github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJ github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= -github.com/jinzhu/gorm v1.9.11 h1:gaHGvE+UnWGlbWG4Y3FUwY1EcZ5n6S9WtqBA/uySMLE= -github.com/jinzhu/gorm v1.9.11/go.mod h1:bu/pK8szGZ2puuErfU0RwyeNdsf3e6nCX/noXaVxkfw= +github.com/jackc/chunkreader v1.0.0 h1:4s39bBR8ByfqH+DKm8rQA3E1LHZWB9XWcrz8fqaZbe0= +github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo= +github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8= +github.com/jackc/chunkreader/v2 v2.0.1/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/pgconn v0.0.0-20190420214824-7e0022ef6ba3/go.mod h1:jkELnwuX+w9qN5YIfX0fl88Ehu4XC3keFuOJJk9pcnA= +github.com/jackc/pgconn v0.0.0-20190824142844-760dd75542eb/go.mod h1:lLjNuW/+OfW9/pnVKPazfWOgNfH2aPem8YQ7ilXGvJE= +github.com/jackc/pgconn v0.0.0-20190831204454-2fabfa3c18b7/go.mod h1:ZJKsE/KZfsUgOEh9hBm+xYTstcNHg7UPMVJqRfQxq4s= +github.com/jackc/pgconn v1.4.0/go.mod h1:Y2O3ZDF0q4mMacyWV3AstPJpeHXWGEetiFttmq5lahk= +github.com/jackc/pgconn v1.5.0/go.mod h1:QeD3lBfpTFe8WUnPZWN5KY/mB8FGMIYRdd8P8Jr0fAI= +github.com/jackc/pgconn v1.5.1-0.20200601181101-fa742c524853/go.mod h1:QeD3lBfpTFe8WUnPZWN5KY/mB8FGMIYRdd8P8Jr0fAI= +github.com/jackc/pgconn v1.8.1 h1:ySBX7Q87vOMqKU2bbmKbUvtYhauDFclYbNDYIE1/h6s= +github.com/jackc/pgconn v1.8.1/go.mod h1:JV6m6b6jhjdmzchES0drzCcYcAHS1OPD5xu3OZ/lE2g= +github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= +github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= +github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2 h1:JVX6jT/XfzNqIjye4717ITLaNwV9mWbJx0dLCpcRzdA= +github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgproto3 v1.1.0 h1:FYYE4yRw+AgI8wXIinMlNjBbp/UitDJwfj5LqqewP1A= +github.com/jackc/pgproto3 v1.1.0/go.mod h1:eR5FA3leWg7p9aeAqi37XOTgTIbkABlvcPB3E5rlc78= +github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190420180111-c116219b62db/go.mod h1:bhq50y+xrl9n5mRYyCBFKkpRVTLYJVWeCc+mEAI3yXA= +github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190609003834-432c2951c711/go.mod h1:uH0AWtUmuShn0bcesswc4aBTWGvw0cAxIJp+6OB//Wg= +github.com/jackc/pgproto3/v2 v2.0.0-rc3/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= +github.com/jackc/pgproto3/v2 v2.0.0-rc3.0.20190831210041-4c03ce451f29/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= +github.com/jackc/pgproto3/v2 v2.0.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgproto3/v2 v2.0.6 h1:b1105ZGEMFe7aCvrT1Cca3VoVb4ZFMaFJLJcg/3zD+8= +github.com/jackc/pgproto3/v2 v2.0.6/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgservicefile v0.0.0-20200307190119-3430c5407db8/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= +github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b h1:C8S2+VttkHFdOOCXJe+YGfa4vHYwlt4Zx+IVXQ97jYg= +github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= +github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01CGwFsrv11mJRHWJ6aifDLfdV3aVjFF0zg= +github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc= +github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw= +github.com/jackc/pgtype v1.2.0/go.mod h1:5m2OfMh1wTK7x+Fk952IDmI4nw3nPrvtQdM0ZT4WpC0= +github.com/jackc/pgtype v1.3.1-0.20200510190516-8cd94a14c75a/go.mod h1:vaogEUkALtxZMCH411K+tKzNpwzCKU+AnPzBKZ+I+Po= +github.com/jackc/pgtype v1.3.1-0.20200606141011-f6355165a91c/go.mod h1:cvk9Bgu/VzJ9/lxTO5R5sf80p0DiucVtN7ZxvaC4GmQ= +github.com/jackc/pgtype v1.7.0 h1:6f4kVsW01QftE38ufBYxKciO6gyioXSC0ABIRLcZrGs= +github.com/jackc/pgtype v1.7.0/go.mod h1:ZnHF+rMePVqDKaOfJVI4Q8IVvAQMryDlDkZnKOI75BE= +github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y= +github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM= +github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc= +github.com/jackc/pgx/v4 v4.5.0/go.mod h1:EpAKPLdnTorwmPUUsqrPxy5fphV18j9q3wrfRXgo+kA= +github.com/jackc/pgx/v4 v4.6.1-0.20200510190926-94ba730bb1e9/go.mod h1:t3/cdRQl6fOLDxqtlyhe9UWgfIi9R8+8v8GKV5TRA/o= +github.com/jackc/pgx/v4 v4.6.1-0.20200606145419-4e5062306904/go.mod h1:ZDaNWkt9sW1JMiNn0kdYBaLelIhw7Pg4qd+Vk6tw7Hg= +github.com/jackc/pgx/v4 v4.11.0 h1:J86tSWd3Y7nKjwT/43xZBvpi04keQWx8gNC2YkdJhZI= +github.com/jackc/pgx/v4 v4.11.0/go.mod h1:i62xJgdrtVDsnL3U8ekyrQXEwGNTRoG7/8r+CIdYfcc= +github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.1.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.1.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= -github.com/jinzhu/now v1.0.1 h1:HjfetcXq097iXP0uoPCdnM4Efp5/9MsM0/M+XOTeR3M= -github.com/jinzhu/now v1.0.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/jinzhu/now v1.1.2 h1:eVKgfIdy9b6zbWBMgFpfDPoAMifwSZagU9HmEU6zgiI= +github.com/jinzhu/now v1.1.2/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= @@ -370,6 +420,7 @@ github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQL github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= @@ -378,11 +429,15 @@ github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= +github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.3.0 h1:/qkRGz8zljWiDcFvgpwUpwIAPu3r07TDvs3Rws+o/pU= +github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= @@ -392,15 +447,20 @@ github.com/magiconair/properties v1.8.4/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPK github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= +github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8= github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= -github.com/mattn/go-sqlite3 v1.11.0 h1:LDdKkqtYlom37fkvqs8rMPFKAMe8+SgjbwZ6ex1/A/Q= -github.com/mattn/go-sqlite3 v1.11.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= @@ -513,14 +573,21 @@ github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqn github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= +github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= +github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= +github.com/shopspring/decimal v0.0.0-20200227202807-02e2044944cc h1:jUIKcSPO9MoMJBbEoyE/RJoE8vz7Mb8AjvifMMwSyvY= +github.com/shopspring/decimal v0.0.0-20200227202807-02e2044944cc/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/sirupsen/logrus v1.7.0 h1:ShrD1U9pZB12TX0cVy0DtePoCH97K8EtX+mg7ZARUtM= @@ -580,6 +647,7 @@ github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= @@ -596,25 +664,32 @@ go.opencensus.io v0.22.6/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= +go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= +go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad h1:DN0cp81fZ3njFcrLCytUHRSUkqBjfTo4Tx9RJTWs0EY= golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 h1:It14KIkyBFYkHkwZ7k45minvA9aorojkyjGk9KJ5B/w= +golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -694,6 +769,8 @@ golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210119194325-5f4716e94777 h1:003p0dJM77cxMSyCPFphvZf/Y5/NXf5fzg6ufd1/Oew= golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 h1:qWPm9rbaAMKs8Bq/9LRpbMqxWRVUAQwMI9fVrssnTfw= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -723,7 +800,9 @@ golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -732,6 +811,7 @@ golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -795,6 +875,7 @@ golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= @@ -802,6 +883,7 @@ golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190823170909-c4a336ef6a2f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -846,6 +928,8 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0 h1:po9/4sTYwZU9lPhi1tOrb4hCv3qrhiQ77LZfGa2OjwY= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -885,7 +969,6 @@ google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6 google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190404172233-64821d5d2107/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -971,6 +1054,7 @@ gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qS gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= +gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.62.0 h1:duBzk771uxoUuOlyRLkHsygud9+5lrlGjdFBb4mSKDU= @@ -993,6 +1077,10 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gorm.io/driver/postgres v1.1.0 h1:afBljg7PtJ5lA6YUWluV2+xovIPhS+YiInuL3kUjrbk= +gorm.io/driver/postgres v1.1.0/go.mod h1:hXQIwafeRjJvUm+OMxcFWyswJ/vevcpPLlGocwAwuqw= +gorm.io/gorm v1.21.9 h1:INieZtn4P2Pw6xPJ8MzT0G4WUOsHq3RhfuDF1M6GW0E= +gorm.io/gorm v1.21.9/go.mod h1:F+OptMscr0P2F2qU97WT1WimdH9GaQPoDW7AYd5i2Y0= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/pkg/common/filters.go b/pkg/common/filters.go index 58ca7916..87b81aeb 100644 --- a/pkg/common/filters.go +++ b/pkg/common/filters.go @@ -2,8 +2,8 @@ package common // Common constants and types for Filtering const ( - DefaultPageOffset = uint32(0) - MaxPageLimit = uint32(50) + DefaultPageOffset = 0 + MaxPageLimit = 50 ) // Common Entity types that can be used on any filters diff --git a/pkg/manager/impl/artifact_manager.go b/pkg/manager/impl/artifact_manager.go index a1f1bebc..85553d1a 100644 --- a/pkg/manager/impl/artifact_manager.go +++ b/pkg/manager/impl/artifact_manager.go @@ -276,7 +276,7 @@ func (m *artifactManager) ListArtifacts(ctx context.Context, request *datacatalo artifact.Data = artifactDataList } - token := strconv.Itoa(int(listInput.Offset) + len(artifactsList)) + token := strconv.Itoa(listInput.Offset + len(artifactsList)) logger.Debugf(ctx, "Listed %v matching artifacts successfully", len(artifactsList)) m.systemMetrics.listSuccessCounter.Inc(ctx) diff --git a/pkg/manager/impl/dataset_manager.go b/pkg/manager/impl/dataset_manager.go index 835d0255..2607f5c3 100644 --- a/pkg/manager/impl/dataset_manager.go +++ b/pkg/manager/impl/dataset_manager.go @@ -182,7 +182,7 @@ func (dm *datasetManager) ListDatasets(ctx context.Context, request *datacatalog return nil, errors.NewCollectedErrors(codes.Internal, transformerErrs) } - token := strconv.Itoa(int(listInput.Offset) + len(datasetList)) + token := strconv.Itoa(listInput.Offset + len(datasetList)) logger.Debugf(ctx, "Listed %v matching datasets successfully", len(datasetList)) dm.systemMetrics.listSuccessCounter.Inc(ctx) diff --git a/pkg/repositories/config/database.go b/pkg/repositories/config/database.go index 7ac5930b..f6c5b26f 100644 --- a/pkg/repositories/config/database.go +++ b/pkg/repositories/config/database.go @@ -1,5 +1,7 @@ package config +import "gorm.io/gorm/logger" + //go:generate pflags DbConfigSection // This struct corresponds to the database section of in the config @@ -12,7 +14,8 @@ type DbConfigSection struct { Password string `json:"password"` PasswordPath string `json:"passwordPath"` // See http://gorm.io/docs/connecting_to_the_database.html for available options passed, in addition to the above. - ExtraOptions string `json:"options"` + ExtraOptions string `json:"options"` + LogLevel logger.LogLevel `json:"log_level"` } // Database config. Contains values necessary to open a database connection. diff --git a/pkg/repositories/config/postgres.go b/pkg/repositories/config/postgres.go index 7e73a694..f8449cca 100644 --- a/pkg/repositories/config/postgres.go +++ b/pkg/repositories/config/postgres.go @@ -3,30 +3,29 @@ package config import ( "fmt" + "gorm.io/gorm/logger" + "github.com/flyteorg/flytestdlib/promutils" - "github.com/jinzhu/gorm" - _ "github.com/jinzhu/gorm/dialects/postgres" // Required to import database driver. + "gorm.io/driver/postgres" + "gorm.io/gorm" ) const Postgres = "postgres" // Generic interface for providing a config necessary to open a database connection. type DbConnectionConfigProvider interface { - // Returns the database type. For instance PostgreSQL or MySQL. - GetType() string - // Returns arguments specific for the database type necessary to open a database connection. - GetArgs() string - // Enables verbose logging. - WithDebugModeEnabled() - // Disables verbose logging. - WithDebugModeDisabled() - // Returns whether verbose logging is enabled or not. - IsDebug() bool + // Returns database dialector + GetDialector() gorm.Dialector + + GetDBConfig() DbConfig + + GetDSN() string } type BaseConfig struct { - IsDebug bool + LogLevel logger.LogLevel `json:"log_level"` + DisableForeignKeyConstraintWhenMigrating bool } // PostgreSQL implementation for DbConnectionConfigProvider. @@ -43,11 +42,7 @@ func NewPostgresConfigProvider(config DbConfig, scope promutils.Scope) DbConnect } } -func (p *PostgresConfigProvider) GetType() string { - return Postgres -} - -func (p *PostgresConfigProvider) GetArgs() string { +func (p *PostgresConfigProvider) GetDSN() string { if p.config.Password == "" { // Switch for development return fmt.Sprintf("host=%s port=%d dbname=%s user=%s sslmode=disable", @@ -57,25 +52,23 @@ func (p *PostgresConfigProvider) GetArgs() string { p.config.Host, p.config.Port, p.config.DbName, p.config.User, p.config.Password, p.config.ExtraOptions) } -func (p *PostgresConfigProvider) WithDebugModeEnabled() { - p.config.IsDebug = true -} - -func (p *PostgresConfigProvider) WithDebugModeDisabled() { - p.config.IsDebug = false +func (p *PostgresConfigProvider) GetDialector() gorm.Dialector { + return postgres.Open(p.GetDSN()) } -func (p *PostgresConfigProvider) IsDebug() bool { - return p.config.IsDebug +func (p *PostgresConfigProvider) GetDBConfig() DbConfig { + return p.config } // Opens a connection to the database specified in the config. // You must call CloseDbConnection at the end of your session! func OpenDbConnection(config DbConnectionConfigProvider) (*gorm.DB, error) { - db, err := gorm.Open(config.GetType(), config.GetArgs()) + db, err := gorm.Open(config.GetDialector(), &gorm.Config{ + Logger: logger.Default.LogMode(config.GetDBConfig().LogLevel), + DisableForeignKeyConstraintWhenMigrating: config.GetDBConfig().DisableForeignKeyConstraintWhenMigrating, + }) if err != nil { return nil, err } - db.LogMode(config.IsDebug()) return db, nil } diff --git a/pkg/repositories/config/postgres_test.go b/pkg/repositories/config/postgres_test.go index c79fe332..02be2cd0 100644 --- a/pkg/repositories/config/postgres_test.go +++ b/pkg/repositories/config/postgres_test.go @@ -3,6 +3,8 @@ package config import ( "testing" + "gorm.io/gorm/logger" + mockScope "github.com/flyteorg/flytestdlib/promutils" "github.com/stretchr/testify/assert" @@ -15,9 +17,15 @@ func TestConstructGormArgs(t *testing.T) { DbName: "postgres", User: "postgres", ExtraOptions: "sslmode=disable", + BaseConfig: BaseConfig{ + LogLevel: 3, + DisableForeignKeyConstraintWhenMigrating: true, + }, }, mockScope.NewTestScope()) - assert.Equal(t, "host=localhost port=5432 dbname=postgres user=postgres sslmode=disable", postgresConfigProvider.GetArgs()) + assert.Equal(t, "host=localhost port=5432 dbname=postgres user=postgres sslmode=disable", postgresConfigProvider.GetDSN()) + assert.Equal(t, logger.LogLevel(3), postgresConfigProvider.GetDBConfig().LogLevel) + assert.Equal(t, true, postgresConfigProvider.GetDBConfig().DisableForeignKeyConstraintWhenMigrating) } func TestConstructGormArgsWithPassword(t *testing.T) { @@ -30,7 +38,7 @@ func TestConstructGormArgsWithPassword(t *testing.T) { ExtraOptions: "sslmode=enable", }, mockScope.NewTestScope()) - assert.Equal(t, "host=localhost port=5432 dbname=postgres user=postgres password=pass sslmode=enable", postgresConfigProvider.GetArgs()) + assert.Equal(t, "host=localhost port=5432 dbname=postgres user=postgres password=pass sslmode=enable", postgresConfigProvider.GetDSN()) } func TestConstructGormArgsWithPasswordNoExtra(t *testing.T) { @@ -42,5 +50,5 @@ func TestConstructGormArgsWithPasswordNoExtra(t *testing.T) { Password: "pass", }, mockScope.NewTestScope()) - assert.Equal(t, "host=localhost port=5432 dbname=postgres user=postgres password=pass ", postgresConfigProvider.GetArgs()) + assert.Equal(t, "host=localhost port=5432 dbname=postgres user=postgres password=pass ", postgresConfigProvider.GetDSN()) } diff --git a/pkg/repositories/errors/postgres.go b/pkg/repositories/errors/postgres.go index 8b663906..d68a9321 100644 --- a/pkg/repositories/errors/postgres.go +++ b/pkg/repositories/errors/postgres.go @@ -4,9 +4,9 @@ import ( "fmt" "github.com/flyteorg/datacatalog/pkg/errors" - "github.com/jinzhu/gorm" "github.com/lib/pq" "google.golang.org/grpc/codes" + "gorm.io/gorm" ) // Postgres error codes diff --git a/pkg/repositories/gormimpl/artifact.go b/pkg/repositories/gormimpl/artifact.go index 757e248d..6b1e445a 100644 --- a/pkg/repositories/gormimpl/artifact.go +++ b/pkg/repositories/gormimpl/artifact.go @@ -3,13 +3,14 @@ package gormimpl import ( "context" + "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/datacatalog" + "github.com/flyteorg/datacatalog/pkg/common" "github.com/flyteorg/datacatalog/pkg/repositories/errors" "github.com/flyteorg/datacatalog/pkg/repositories/interfaces" "github.com/flyteorg/datacatalog/pkg/repositories/models" - "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/datacatalog" "github.com/flyteorg/flytestdlib/promutils" - "github.com/jinzhu/gorm" + "gorm.io/gorm" ) type artifactRepo struct { @@ -65,19 +66,20 @@ func (h *artifactRepo) Get(ctx context.Context, in models.ArtifactKey) (models.A ) if result.Error != nil { + if result.Error.Error() == gorm.ErrRecordNotFound.Error() { + return models.Artifact{}, errors.GetMissingEntityError("Artifact", &datacatalog.Artifact{ + Dataset: &datacatalog.DatasetID{ + Project: in.DatasetProject, + Domain: in.DatasetDomain, + Name: in.DatasetName, + Version: in.DatasetVersion, + }, + Id: in.ArtifactID, + }) + } + return models.Artifact{}, h.errorTransformer.ToDataCatalogError(result.Error) } - if result.RecordNotFound() { - return models.Artifact{}, errors.GetMissingEntityError("Artifact", &datacatalog.Artifact{ - Dataset: &datacatalog.DatasetID{ - Project: in.DatasetProject, - Domain: in.DatasetDomain, - Name: in.DatasetName, - Version: in.DatasetVersion, - }, - Id: in.ArtifactID, - }) - } return artifact, nil } diff --git a/pkg/repositories/gormimpl/artifact_test.go b/pkg/repositories/gormimpl/artifact_test.go index 027bb0f8..49292fb8 100644 --- a/pkg/repositories/gormimpl/artifact_test.go +++ b/pkg/repositories/gormimpl/artifact_test.go @@ -117,21 +117,22 @@ func TestCreateArtifact(t *testing.T) { // Only match on queries that append expected filters GlobalMock.NewMock().WithQuery( - `INSERT INTO "artifacts" ("created_at","updated_at","deleted_at","dataset_project","dataset_name","dataset_domain","dataset_version","artifact_id","dataset_uuid","serialized_metadata") VALUES (?,?,?,?,?,?,?,?,?,?)`).WithCallback( + `INSERT INTO "artifacts" ("created_at","updated_at","deleted_at","dataset_project","dataset_name","dataset_domain","dataset_version","artifact_id","dataset_uuid","serialized_metadata") VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10)`).WithCallback( func(s string, values []driver.NamedValue) { artifactCreated = true }, ) GlobalMock.NewMock().WithQuery( - `INSERT INTO "artifact_data" ("created_at","updated_at","deleted_at","dataset_project","dataset_name","dataset_domain","dataset_version","artifact_id","name","location") VALUES (?,?,?,?,?,?,?,?,?,?)`).WithCallback( + `INSERT INTO "artifact_data" ("created_at","updated_at","deleted_at","dataset_project","dataset_name","dataset_domain","dataset_version","artifact_id","name","location") VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10),($11,$12,$13,$14,$15,$16,$17,$18,$19,$20) ON CONFLICT ("dataset_project","dataset_name","dataset_domain","dataset_version","artifact_id","name") DO UPDATE SET "dataset_project"="excluded"."dataset_project","dataset_name"="excluded"."dataset_name","dataset_domain"="excluded"."dataset_domain","dataset_version"="excluded"."dataset_version","artifact_id"="excluded"."artifact_id"`).WithCallback( func(s string, values []driver.NamedValue) { - numArtifactDataCreated++ + // Batch insert + numArtifactDataCreated += 2 }, ) GlobalMock.NewMock().WithQuery( - `INSERT INTO "partitions" ("created_at","updated_at","deleted_at","dataset_uuid","key","value","artifact_id") VALUES (?,?,?,?,?,?,?)`).WithCallback( + `INSERT INTO "partitions" ("created_at","updated_at","deleted_at","dataset_uuid","key","value","artifact_id") VALUES ($1,$2,$3,$4,$5,$6,$7) ON CONFLICT ("dataset_uuid","key","value","artifact_id") DO UPDATE SET "artifact_id"="excluded"."artifact_id"`).WithCallback( func(s string, values []driver.NamedValue) { numPartitionsCreated++ }, @@ -175,13 +176,13 @@ func TestGetArtifact(t *testing.T) { // Only match on queries that append expected filters GlobalMock.NewMock().WithQuery( - `SELECT * FROM "artifacts" WHERE "artifacts"."deleted_at" IS NULL AND (("artifacts"."dataset_project" = testProject) AND ("artifacts"."dataset_name" = testName) AND ("artifacts"."dataset_domain" = testDomain) AND ("artifacts"."dataset_version" = testVersion) AND ("artifacts"."artifact_id" = 123)) ORDER BY artifacts.created_at DESC,"artifacts"."dataset_project" ASC LIMIT 1`).WithReply(expectedArtifactResponse) + `SELECT * FROM "artifacts" WHERE "artifacts"."dataset_project" = $1 AND "artifacts"."dataset_name" = $2 AND "artifacts"."dataset_domain" = $3 AND "artifacts"."dataset_version" = $4 AND "artifacts"."artifact_id" = $5 ORDER BY artifacts.created_at DESC,"artifacts"."created_at" LIMIT 1%!!(string=123)!(string=testVersion)!(string=testDomain)!(string=testName)(EXTRA string=testProject)`).WithReply(expectedArtifactResponse) GlobalMock.NewMock().WithQuery( - `SELECT * FROM "artifact_data" WHERE "artifact_data"."deleted_at" IS NULL AND ((("dataset_project","dataset_name","dataset_domain","dataset_version","artifact_id") IN ((testProject,testName,testDomain,testVersion,123)))) ORDER BY "artifact_data"."dataset_project" ASC`).WithReply(expectedArtifactDataResponse) + `SELECT * FROM "artifact_data" WHERE ("artifact_data"."dataset_project","artifact_data"."dataset_name","artifact_data"."dataset_domain","artifact_data"."dataset_version","artifact_data"."artifact_id") IN (($1,$2,$3,$4,$5))%!!(string=123)!(string=testVersion)!(string=testDomain)!(string=testName)(EXTRA string=testProject)`).WithReply(expectedArtifactDataResponse) GlobalMock.NewMock().WithQuery( - `SELECT * FROM "partitions" WHERE "partitions"."deleted_at" IS NULL AND (("artifact_id" IN (123))) ORDER BY partitions.created_at ASC,"partitions"."dataset_uuid" ASC`).WithReply(expectedPartitionResponse) + `SELECT * FROM "partitions" WHERE "partitions"."artifact_id" = $1 ORDER BY partitions.created_at ASC%!(EXTRA string=123)`).WithReply(expectedPartitionResponse) GlobalMock.NewMock().WithQuery( - `SELECT * FROM "tags" WHERE "tags"."deleted_at" IS NULL AND ((("artifact_id","dataset_uuid") IN ((123,test-uuid)))) ORDER BY "tags"."dataset_project" ASC`).WithReply(expectedTagResponse) + `SELECT * FROM "tags" WHERE ("tags"."artifact_id","tags"."dataset_uuid") IN (($1,$2))%!!(string=test-uuid)(EXTRA string=123)`).WithReply(expectedTagResponse) getInput := models.ArtifactKey{ DatasetProject: artifact.DatasetProject, DatasetDomain: artifact.DatasetDomain, @@ -217,13 +218,13 @@ func TestGetArtifactByID(t *testing.T) { // Only match on queries that append expected filters GlobalMock.NewMock().WithQuery( - `SELECT * FROM "artifacts" WHERE "artifacts"."deleted_at" IS NULL AND (("artifacts"."artifact_id" = 123)) ORDER BY artifacts.created_at DESC,"artifacts"."dataset_project" ASC LIMIT 1`).WithReply(expectedArtifactResponse) + `SELECT * FROM "artifacts" WHERE "artifacts"."artifact_id" = $1 ORDER BY artifacts.created_at DESC,"artifacts"."created_at" LIMIT 1%!(EXTRA string=123)`).WithReply(expectedArtifactResponse) GlobalMock.NewMock().WithQuery( - `SELECT * FROM "artifact_data" WHERE "artifact_data"."deleted_at" IS NULL AND ((("dataset_project","dataset_name","dataset_domain","dataset_version","artifact_id") IN ((testProject,testName,testDomain,testVersion,123)))) ORDER BY "artifact_data"."dataset_project" ASC`).WithReply(expectedArtifactDataResponse) + `SELECT * FROM "artifact_data" WHERE ("artifact_data"."dataset_project","artifact_data"."dataset_name","artifact_data"."dataset_domain","artifact_data"."dataset_version","artifact_data"."artifact_id") IN (($1,$2,$3,$4,$5))%!!(string=123)!(string=testVersion)!(string=testDomain)!(string=testName)(EXTRA string=testProject)`).WithReply(expectedArtifactDataResponse) GlobalMock.NewMock().WithQuery( - `SELECT * FROM "partitions" WHERE "partitions"."deleted_at" IS NULL AND (("artifact_id" IN (123))) ORDER BY partitions.created_at ASC,"partitions"."dataset_uuid" ASC`).WithReply(expectedPartitionResponse) + `SELECT * FROM "partitions" WHERE "partitions"."artifact_id" = $1 ORDER BY partitions.created_at ASC%!(EXTRA string=123)`).WithReply(expectedPartitionResponse) GlobalMock.NewMock().WithQuery( - `SELECT * FROM "tags" WHERE "tags"."deleted_at" IS NULL AND ((("artifact_id","dataset_uuid") IN ((123,test-uuid)))) ORDER BY "tags"."dataset_project" ASC`).WithReply(expectedTagResponse) + `SELECT * FROM "tags" WHERE ("tags"."artifact_id","tags"."dataset_uuid") IN (($1,$2))%!!(string=test-uuid)(EXTRA string=123)`).WithReply(expectedTagResponse) getInput := models.ArtifactKey{ ArtifactID: artifact.ArtifactID, } @@ -265,7 +266,7 @@ func TestCreateArtifactAlreadyExists(t *testing.T) { // Only match on queries that append expected filters GlobalMock.NewMock().WithQuery( - `INSERT INTO "artifacts" ("created_at","updated_at","deleted_at","dataset_project","dataset_name","dataset_domain","dataset_version","artifact_id","dataset_uuid","serialized_metadata") VALUES (?,?,?,?,?,?,?,?,?,?)`).WithError( + `INSERT INTO "artifacts" ("created_at","updated_at","deleted_at","dataset_project","dataset_name","dataset_domain","dataset_version","artifact_id","dataset_uuid","serialized_metadata") VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10)`).WithError( getAlreadyExistsErr(), ) @@ -289,13 +290,13 @@ func TestListArtifactsWithPartition(t *testing.T) { expectedPartitionResponse := getDBPartitionResponse(artifact) expectedTagResponse := getDBTagResponse(artifact) GlobalMock.NewMock().WithQuery( - `SELECT "artifacts".* FROM "artifacts" JOIN partitions partitions0 ON artifacts.artifact_id = partitions0.artifact_id WHERE "artifacts"."deleted_at" IS NULL AND ((partitions0.key = val1) AND (partitions0.val = val2) AND (artifacts.dataset_uuid = test-uuid)) ORDER BY artifacts.created_at desc LIMIT 10 OFFSET 10`).WithReply(expectedArtifactResponse) + `SELECT "artifacts"."created_at","artifacts"."updated_at","artifacts"."deleted_at","artifacts"."dataset_project","artifacts"."dataset_name","artifacts"."dataset_domain","artifacts"."dataset_version","artifacts"."artifact_id","artifacts"."dataset_uuid","artifacts"."serialized_metadata" FROM "artifacts" JOIN partitions partitions0 ON artifacts.artifact_id = partitions0.artifact_id WHERE partitions0.key = $1 AND partitions0.val = $2 AND artifacts.dataset_uuid = $3 ORDER BY artifacts.created_at desc LIMIT 10 OFFSET 10%!!(string=test-uuid)!(string=val2)(EXTRA string=val1)`).WithReply(expectedArtifactResponse) GlobalMock.NewMock().WithQuery( - `SELECT * FROM "artifact_data" WHERE "artifact_data"."deleted_at" IS NULL AND ((("dataset_project","dataset_name","dataset_domain","dataset_version","artifact_id") IN ((testProject,testName,testDomain,testVersion,123))))`).WithReply(expectedArtifactDataResponse) + `SELECT * FROM "artifact_data" WHERE ("artifact_data"."dataset_project","artifact_data"."dataset_name","artifact_data"."dataset_domain","artifact_data"."dataset_version","artifact_data"."artifact_id") IN (($1,$2,$3,$4,$5))%!!(string=123)!(string=testVersion)!(string=testDomain)!(string=testName)(EXTRA string=testProject)`).WithReply(expectedArtifactDataResponse) GlobalMock.NewMock().WithQuery( - `SELECT * FROM "partitions" WHERE "partitions"."deleted_at" IS NULL AND (("artifact_id" IN (123))) ORDER BY partitions.created_at ASC`).WithReply(expectedPartitionResponse) + `SELECT * FROM "partitions" WHERE "partitions"."artifact_id" = $1 ORDER BY partitions.created_at ASC%!(EXTRA string=123)`).WithReply(expectedPartitionResponse) GlobalMock.NewMock().WithQuery( - `SELECT * FROM "tags" WHERE "tags"."deleted_at" IS NULL AND ((("artifact_id","dataset_uuid") IN ((123,test-uuid))))`).WithReply(expectedTagResponse) + `SELECT * FROM "tags" WHERE ("tags"."artifact_id","tags"."dataset_uuid") IN (($1,$2))%!!(string=test-uuid)(EXTRA string=123)`).WithReply(expectedTagResponse) artifactRepo := NewArtifactRepo(utils.GetDbForTest(t), errors.NewPostgresErrorTransformer(), promutils.NewTestScope()) listInput := models.ListModelsInput{ @@ -333,11 +334,11 @@ func TestListArtifactsNoPartitions(t *testing.T) { expectedPartitionResponse := make([]map[string]interface{}, 0) GlobalMock.NewMock().WithQuery( - `SELECT * FROM "artifacts" WHERE "artifacts"."deleted_at" IS NULL AND ((artifacts.dataset_uuid = test-uuid)) LIMIT 10 OFFSET 10`).WithReply(expectedArtifactResponse) + `SELECT * FROM "artifacts" WHERE artifacts.dataset_uuid = $1 LIMIT 10 OFFSET 10%!(EXTRA string=test-uuid)`).WithReply(expectedArtifactResponse) GlobalMock.NewMock().WithQuery( - `SELECT * FROM "artifact_data" WHERE "artifact_data"."deleted_at" IS NULL AND ((("dataset_project","dataset_name","dataset_domain","dataset_version","artifact_id") IN ((testProject,testName,testDomain,testVersion,123))))`).WithReply(expectedArtifactDataResponse) + `SELECT * FROM "artifact_data" WHERE ("artifact_data"."dataset_project","artifact_data"."dataset_name","artifact_data"."dataset_domain","artifact_data"."dataset_version","artifact_data"."artifact_id") IN (($1,$2,$3,$4,$5))%!!(string=123)!(string=testVersion)!(string=testDomain)!(string=testName)(EXTRA string=testProject)`).WithReply(expectedArtifactDataResponse) GlobalMock.NewMock().WithQuery( - `SELECT * FROM "partitions" WHERE "partitions"."deleted_at" IS NULL AND (("artifact_id" IN (123)))`).WithReply(expectedPartitionResponse) + `SELECT * FROM "partitions" WHERE "partitions"."artifact_id" = $1 ORDER BY partitions.created_at ASC%!(EXTRA string=123)`).WithReply(expectedPartitionResponse) artifactRepo := NewArtifactRepo(utils.GetDbForTest(t), errors.NewPostgresErrorTransformer(), promutils.NewTestScope()) listInput := models.ListModelsInput{ diff --git a/pkg/repositories/gormimpl/dataset.go b/pkg/repositories/gormimpl/dataset.go index 90d20c4d..c7b48158 100644 --- a/pkg/repositories/gormimpl/dataset.go +++ b/pkg/repositories/gormimpl/dataset.go @@ -3,14 +3,15 @@ package gormimpl import ( "context" + idl_datacatalog "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/datacatalog" + "github.com/flyteorg/datacatalog/pkg/common" "github.com/flyteorg/datacatalog/pkg/repositories/errors" "github.com/flyteorg/datacatalog/pkg/repositories/interfaces" "github.com/flyteorg/datacatalog/pkg/repositories/models" - idl_datacatalog "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/datacatalog" "github.com/flyteorg/flytestdlib/logger" "github.com/flyteorg/flytestdlib/promutils" - "github.com/jinzhu/gorm" + "gorm.io/gorm" ) type dataSetRepo struct { @@ -51,16 +52,17 @@ func (h *dataSetRepo) Get(ctx context.Context, in models.DatasetKey) (models.Dat if result.Error != nil { logger.Debugf(ctx, "Unable to find Dataset: [%+v], err: %v", in, result.Error) + + if result.Error.Error() == gorm.ErrRecordNotFound.Error() { + return models.Dataset{}, errors.GetMissingEntityError("Dataset", &idl_datacatalog.DatasetID{ + Project: in.Project, + Domain: in.Domain, + Name: in.Name, + Version: in.Version, + }) + } return models.Dataset{}, h.errorTransformer.ToDataCatalogError(result.Error) } - if result.RecordNotFound() { - return models.Dataset{}, errors.GetMissingEntityError("Dataset", &idl_datacatalog.DatasetID{ - Project: in.Project, - Domain: in.Domain, - Name: in.Name, - Version: in.Version, - }) - } return ds, nil } diff --git a/pkg/repositories/gormimpl/dataset_test.go b/pkg/repositories/gormimpl/dataset_test.go index 41d56b51..2f2ef8e9 100644 --- a/pkg/repositories/gormimpl/dataset_test.go +++ b/pkg/repositories/gormimpl/dataset_test.go @@ -1,9 +1,9 @@ package gormimpl import ( - "testing" - "context" + "testing" + "time" mocket "github.com/Selvatico/go-mocket" "google.golang.org/grpc/codes" @@ -40,6 +40,11 @@ func getTestDataset() models.Dataset { {Name: "key1"}, {Name: "key2"}, }, + BaseModel: models.BaseModel{ + CreatedAt: time.Unix(111, 0), + UpdatedAt: time.Unix(111, 0), + DeletedAt: nil, + }, } } @@ -82,7 +87,7 @@ func TestCreateDatasetNoPartitions(t *testing.T) { // Only match on queries that append expected filters GlobalMock.NewMock().WithQuery( - `INSERT INTO "datasets" ("created_at","updated_at","deleted_at","project","name","domain","version","serialized_metadata") VALUES (?,?,?,?,?,?,?,?)`).WithCallback( + `INSERT INTO "datasets" ("created_at","updated_at","deleted_at","project","name","domain","version","serialized_metadata") VALUES ($1,$2,$3,$4,$5,$6,$7,$8)`).WithCallback( func(s string, values []driver.NamedValue) { assert.EqualValues(t, dataset.Project, values[3].Value) assert.EqualValues(t, dataset.Name, values[4].Value) @@ -110,7 +115,7 @@ func TestCreateDataset(t *testing.T) { // Only match on queries that append expected filters GlobalMock.NewMock().WithQuery( - `INSERT INTO "datasets" ("created_at","updated_at","deleted_at","project","name","domain","version","serialized_metadata") VALUES (?,?,?,?,?,?,?,?)`).WithCallback( + `INSERT INTO "datasets" ("created_at","updated_at","deleted_at","project","name","domain","version","serialized_metadata") VALUES ($1,$2,$3,$4,$5,$6,$7,$8)`).WithCallback( func(s string, values []driver.NamedValue) { assert.EqualValues(t, dataset.Project, values[3].Value) assert.EqualValues(t, dataset.Name, values[4].Value) @@ -119,17 +124,14 @@ func TestCreateDataset(t *testing.T) { assert.EqualValues(t, dataset.SerializedMetadata, values[7].Value) datasetCreated = true }, - ) - - GlobalMock.NewMock().WithQuery( - `SELECT "uuid" FROM "datasets" WHERE (project = testProject) AND (name = testName) AND (domain = testDomain) AND (version = testVersion)`).WithReply([]map[string]interface{}{{"uuid": getDatasetUUID()}}) + ).WithReply([]map[string]interface{}{{"dataset_uuid": getDatasetUUID()}}) GlobalMock.NewMock().WithQuery( - `INSERT INTO "partition_keys" ("created_at","updated_at","deleted_at","dataset_uuid","name") VALUES (?,?,?,?,?)`).WithCallback( + `INSERT INTO "partition_keys" ("created_at","updated_at","deleted_at","dataset_uuid","name") VALUES ($1,$2,$3,$4,$5),($6,$7,$8,$9,$10) ON CONFLICT ("dataset_uuid","name") DO UPDATE SET "dataset_uuid"="excluded"."dataset_uuid"`).WithCallback( func(s string, values []driver.NamedValue) { assert.EqualValues(t, getDatasetUUID(), values[3].Value) assert.EqualValues(t, dataset.PartitionKeys[insertKeyQueryNum].Name, values[4].Value) - insertKeyQueryNum++ + insertKeyQueryNum += 2 // batch insertion }, ) @@ -157,7 +159,7 @@ func TestGetDataset(t *testing.T) { GlobalMock.Logging = true // Only match on queries that append expected filters - GlobalMock.NewMock().WithQuery(`SELECT * FROM "datasets" WHERE "datasets"."deleted_at" IS NULL AND (("datasets"."project" = testProject) AND ("datasets"."name" = testName) AND ("datasets"."domain" = testDomain) AND ("datasets"."version" = testVersion)) ORDER BY "datasets"."project" ASC LIMIT 1`).WithReply(expectedDatasetResponse) + GlobalMock.NewMock().WithQuery(`SELECT * FROM "datasets" WHERE "datasets"."project" = $1 AND "datasets"."name" = $2 AND "datasets"."domain" = $3 AND "datasets"."version" = $4 ORDER BY "datasets"."created_at" LIMIT 1`).WithReply(expectedDatasetResponse) expectedPartitionKeyResponse := make([]map[string]interface{}, 0) samplePartitionKey := make(map[string]interface{}) @@ -165,7 +167,7 @@ func TestGetDataset(t *testing.T) { samplePartitionKey["dataset_uuid"] = getDatasetUUID() expectedPartitionKeyResponse = append(expectedPartitionKeyResponse, samplePartitionKey, samplePartitionKey) - GlobalMock.NewMock().WithQuery(`SELECT * FROM "partition_keys" WHERE "partition_keys"."deleted_at" IS NULL AND (("dataset_uuid" IN (test-uuid))) ORDER BY partition_keys.created_at ASC,"partition_keys"."dataset_uuid" ASC`).WithReply(expectedPartitionKeyResponse) + GlobalMock.NewMock().WithQuery(`SELECT * FROM "partition_keys" WHERE "partition_keys"."dataset_uuid" = $1 ORDER BY partition_keys.created_at ASC`).WithReply(expectedPartitionKeyResponse) datasetRepo := NewDatasetRepo(utils.GetDbForTest(t), errors.NewPostgresErrorTransformer(), promutils.NewTestScope()) actualDataset, err := datasetRepo.Get(context.Background(), dataset.DatasetKey) assert.NoError(t, err) @@ -196,7 +198,7 @@ func TestGetDatasetWithUUID(t *testing.T) { GlobalMock.Logging = true // Only match on queries that append expected filters - GlobalMock.NewMock().WithQuery(`SELECT * FROM "datasets" WHERE "datasets"."deleted_at" IS NULL AND (("datasets"."uuid" = test-uuid)) ORDER BY "datasets"."project" ASC LIMIT 1`).WithReply(expectedResponse) + GlobalMock.NewMock().WithQuery(`SELECT * FROM "datasets" WHERE "datasets"."uuid" = $1 ORDER BY "datasets"."created_at" LIMIT 1%!(EXTRA string=test-uuid)`).WithReply(expectedResponse) datasetRepo := NewDatasetRepo(utils.GetDbForTest(t), errors.NewPostgresErrorTransformer(), promutils.NewTestScope()) actualDataset, err := datasetRepo.Get(context.Background(), dataset.DatasetKey) @@ -235,7 +237,7 @@ func TestCreateDatasetAlreadyExists(t *testing.T) { // Only match on queries that append expected filters GlobalMock.NewMock().WithQuery( - `INSERT INTO "datasets" ("created_at","updated_at","deleted_at","project","name","domain","version","serialized_metadata") VALUES (?,?,?,?,?,?,?,?)`).WithError( + `INSERT INTO "datasets" ("created_at","updated_at","deleted_at","project","name","domain","version","serialized_metadata") VALUES ($1,$2,$3,$4,$5,$6,$7,$8)`).WithError( getAlreadyExistsErr(), ) @@ -256,10 +258,10 @@ func TestListDatasets(t *testing.T) { expectedDatasetDBResponse := getDBDatasetResponse(dataset) GlobalMock.NewMock().WithQuery( - `SELECT * FROM "datasets" WHERE "datasets"."deleted_at" IS NULL LIMIT 10 OFFSET 0`).WithReply(expectedDatasetDBResponse) + `SELECT * FROM "datasets" LIMIT 10`).WithReply(expectedDatasetDBResponse) expectedPartitionKeyResponse := getDBPartitionKeysResponse([]models.Dataset{dataset}) - GlobalMock.NewMock().WithQuery(`SELECT * FROM "partition_keys" WHERE "partition_keys"."deleted_at" IS NULL AND (("dataset_uuid" IN (test-uuid)))`).WithReply(expectedPartitionKeyResponse) + GlobalMock.NewMock().WithQuery(`SELECT * FROM "partition_keys" WHERE "partition_keys"."dataset_uuid" = $1`).WithReply(expectedPartitionKeyResponse) datasetRepo := NewDatasetRepo(utils.GetDbForTest(t), errors.NewPostgresErrorTransformer(), promutils.NewTestScope()) listInput := models.ListModelsInput{ Limit: 10, @@ -284,10 +286,10 @@ func TestListDatasetWithFilter(t *testing.T) { expectedDatasetDBResponse := getDBDatasetResponse(dataset) GlobalMock.NewMock().WithQuery( - `SELECT * FROM "datasets" WHERE "datasets"."deleted_at" IS NULL AND ((datasets.project = p) AND (datasets.domain = d)) ORDER BY datasets.created_at desc LIMIT 10 OFFSET 10`).WithReply(expectedDatasetDBResponse) + `SELECT * FROM "datasets" WHERE datasets.project = $1 AND datasets.domain = $2 ORDER BY datasets.created_at desc LIMIT 10 OFFSET 10`).WithReply(expectedDatasetDBResponse) expectedPartitionKeyResponse := getDBPartitionKeysResponse([]models.Dataset{dataset}) - GlobalMock.NewMock().WithQuery(`SELECT * FROM "partition_keys" WHERE "partition_keys"."deleted_at" IS NULL AND (("dataset_uuid" IN (test-uuid)))`).WithReply(expectedPartitionKeyResponse) + GlobalMock.NewMock().WithQuery(`SELECT * FROM "partition_keys" WHERE "partition_keys"."dataset_uuid" = $1%!(EXTRA string=test-uuid)`).WithReply(expectedPartitionKeyResponse) datasetRepo := NewDatasetRepo(utils.GetDbForTest(t), errors.NewPostgresErrorTransformer(), promutils.NewTestScope()) listInput := models.ListModelsInput{ diff --git a/pkg/repositories/gormimpl/list.go b/pkg/repositories/gormimpl/list.go index 9e47b956..a231c8a1 100644 --- a/pkg/repositories/gormimpl/list.go +++ b/pkg/repositories/gormimpl/list.go @@ -3,10 +3,13 @@ package gormimpl import ( "fmt" + errors2 "github.com/flyteorg/datacatalog/pkg/errors" + "github.com/flyteorg/datacatalog/pkg/common" "github.com/flyteorg/datacatalog/pkg/repositories/errors" "github.com/flyteorg/datacatalog/pkg/repositories/models" - "github.com/jinzhu/gorm" + "google.golang.org/grpc/codes" + "gorm.io/gorm" ) const ( @@ -20,6 +23,15 @@ var entityToModel = map[common.Entity]interface{}{ common.Tag: models.Tag{}, } +func getTableName(tx *gorm.DB, model interface{}) (string, error) { + stmt := gorm.Statement{DB: tx} + + if err := stmt.Parse(model); err != nil { + return "", errors2.NewDataCatalogError(codes.InvalidArgument, err.Error()) + } + return stmt.Schema.Table, nil +} + // Apply the list query on the source model. This method will apply the necessary joins, filters and // pagination on the database for the given ListModelInputs. func applyListModelsInput(tx *gorm.DB, sourceEntity common.Entity, in models.ListModelsInput) (*gorm.DB, error) { @@ -28,14 +40,21 @@ func applyListModelsInput(tx *gorm.DB, sourceEntity common.Entity, in models.Lis return nil, errors.GetInvalidEntityError(sourceEntity) } - sourceTableName := tx.NewScope(sourceModel).TableName() + sourceTableName, err := getTableName(tx, sourceModel) + if err != nil { + return nil, err + } + for modelIndex, modelFilter := range in.ModelFilters { entity := modelFilter.Entity filterModel, ok := entityToModel[entity] if !ok { return nil, errors.GetInvalidEntityError(entity) } - tableName := tx.NewScope(filterModel).TableName() + tableName, err := getTableName(tx, filterModel) + if err != nil { + return nil, err + } tableAlias := tableName // Optionally add the join condition if the entity we need isn't the source diff --git a/pkg/repositories/gormimpl/list_test.go b/pkg/repositories/gormimpl/list_test.go index b795afe9..2220636f 100644 --- a/pkg/repositories/gormimpl/list_test.go +++ b/pkg/repositories/gormimpl/list_test.go @@ -20,13 +20,12 @@ func TestApplyFilter(t *testing.T) { validInputApply := false GlobalMock.NewMock().WithQuery( - `SELECT "artifacts".* FROM "artifacts"`).WithCallback( + `SELECT "artifacts"."created_at","artifacts"."updated_at","artifacts"."deleted_at","artifacts"."dataset_project","artifacts"."dataset_name","artifacts"."dataset_domain","artifacts"."dataset_version","artifacts"."artifact_id","artifacts"."dataset_uuid","artifacts"."serialized_metadata" FROM "artifacts"`).WithCallback( func(s string, values []driver.NamedValue) { // separate the regex matching because the joins reorder on different test runs validInputApply = strings.Contains(s, `JOIN tags tags1 ON artifacts.artifact_id = tags1.artifact_id`) && strings.Contains(s, `JOIN partitions partitions0 ON artifacts.artifact_id = partitions0.artifact_id`) && - strings.Contains(s, `WHERE "artifacts"."deleted_at" IS NULL AND `+ - `((partitions0.key1 = val1) AND (partitions0.key2 = val2) AND (tags1.tag_name = special)) `+ + strings.Contains(s, `WHERE partitions0.key1 = $1 AND partitions0.key2 = $2 AND tags1.tag_name = $3 `+ `ORDER BY artifacts.created_at desc LIMIT 10 OFFSET 10`) }) @@ -67,7 +66,7 @@ func TestApplyFilterEmpty(t *testing.T) { validInputApply := false GlobalMock.NewMock().WithQuery( - `SELECT * FROM "artifacts" WHERE "artifacts"."deleted_at" IS NULL LIMIT 10 OFFSET 10`).WithCallback( + `SELECT * FROM "artifacts" LIMIT 10 OFFSET 10`).WithCallback( func(s string, values []driver.NamedValue) { // separate the regex matching because the joins reorder on different test runs validInputApply = true @@ -84,3 +83,11 @@ func TestApplyFilterEmpty(t *testing.T) { tx.Find(models.Artifact{}) assert.True(t, validInputApply) } + +func TestGetTableErr(t *testing.T) { + testDB := utils.GetDbForTest(t) + + tableName, err := getTableName(testDB, "") + assert.Error(t, err) + assert.Equal(t, "", tableName) +} diff --git a/pkg/repositories/gormimpl/tag.go b/pkg/repositories/gormimpl/tag.go index 8e40de84..24c7750e 100644 --- a/pkg/repositories/gormimpl/tag.go +++ b/pkg/repositories/gormimpl/tag.go @@ -3,12 +3,13 @@ package gormimpl import ( "context" + idl_datacatalog "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/datacatalog" + "github.com/flyteorg/datacatalog/pkg/repositories/errors" "github.com/flyteorg/datacatalog/pkg/repositories/interfaces" "github.com/flyteorg/datacatalog/pkg/repositories/models" - idl_datacatalog "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/datacatalog" "github.com/flyteorg/flytestdlib/promutils" - "github.com/jinzhu/gorm" + "gorm.io/gorm" ) type tagRepo struct { @@ -54,13 +55,13 @@ func (h *tagRepo) Get(ctx context.Context, in models.TagKey) (models.Tag, error) }) if result.Error != nil { + if result.Error.Error() == gorm.ErrRecordNotFound.Error() { + return models.Tag{}, errors.GetMissingEntityError("Tag", &idl_datacatalog.Tag{ + Name: tag.TagName, + }) + } return models.Tag{}, h.errorTransformer.ToDataCatalogError(result.Error) } - if result.RecordNotFound() { - return models.Tag{}, errors.GetMissingEntityError("Tag", &idl_datacatalog.Tag{ - Name: tag.TagName, - }) - } return tag, nil } diff --git a/pkg/repositories/gormimpl/tag_test.go b/pkg/repositories/gormimpl/tag_test.go index 2c7526e9..c7c9f3b1 100644 --- a/pkg/repositories/gormimpl/tag_test.go +++ b/pkg/repositories/gormimpl/tag_test.go @@ -3,6 +3,8 @@ package gormimpl import ( "testing" + "gorm.io/gorm" + "context" mocket "github.com/Selvatico/go-mocket" @@ -51,7 +53,7 @@ func TestCreateTag(t *testing.T) { // Only match on queries that append expected filters GlobalMock.NewMock().WithQuery( - `INSERT INTO "tags" ("created_at","updated_at","deleted_at","dataset_project","dataset_name","dataset_domain","dataset_version","tag_name","artifact_id","dataset_uuid") VALUES (?,?,?,?,?,?,?,?,?,?)`).WithCallback( + `INSERT INTO "tags" ("created_at","updated_at","deleted_at","dataset_project","dataset_name","dataset_domain","dataset_version","tag_name","artifact_id","dataset_uuid") VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10)`).WithCallback( func(s string, values []driver.NamedValue) { tagCreated = true }, @@ -71,15 +73,15 @@ func TestGetTag(t *testing.T) { // Only match on queries that append expected filters GlobalMock.NewMock().WithQuery( - `SELECT * FROM "tags" WHERE "tags"."deleted_at" IS NULL AND (("tags"."dataset_project" = testProject) AND ("tags"."dataset_name" = testName) AND ("tags"."dataset_domain" = testDomain) AND ("tags"."dataset_version" = testVersion) AND ("tags"."tag_name" = test-tag)) ORDER BY tags.created_at DESC,"tags"."dataset_project" ASC LIMIT 1`).WithReply(getDBTagResponse(artifact)) + `SELECT * FROM "tags" WHERE "tags"."dataset_project" = $1 AND "tags"."dataset_name" = $2 AND "tags"."dataset_domain" = $3 AND "tags"."dataset_version" = $4 AND "tags"."tag_name" = $5 ORDER BY tags.created_at DESC,"tags"."created_at" LIMIT 1%!!(string=test-tag)!(string=testVersion)!(string=testDomain)!(string=testName)(EXTRA string=testProject)`).WithReply(getDBTagResponse(artifact)) GlobalMock.NewMock().WithQuery( - `SELECT * FROM "artifacts" WHERE "artifacts"."deleted_at" IS NULL AND ((("dataset_project","dataset_name","dataset_domain","dataset_version","artifact_id") IN ((testProject,testName,testDomain,testVersion,123))))`).WithReply(getDBArtifactResponse(artifact)) + `SELECT * FROM "artifacts" WHERE ("artifacts"."dataset_project","artifacts"."dataset_name","artifacts"."dataset_domain","artifacts"."dataset_version","artifacts"."artifact_id") IN (($1,$2,$3,$4,$5))%!!(string=123)!(string=testVersion)!(string=testDomain)!(string=testName)(EXTRA string=testProject)`).WithReply(getDBArtifactResponse(artifact)) GlobalMock.NewMock().WithQuery( - `SELECT * FROM "artifact_data" WHERE "artifact_data"."deleted_at" IS NULL AND ((("dataset_project","dataset_name","dataset_domain","dataset_version","artifact_id") IN ((testProject,testName,testDomain,testVersion,123))))`).WithReply(getDBArtifactDataResponse(artifact)) + `SELECT * FROM "artifact_data" WHERE ("artifact_data"."dataset_project","artifact_data"."dataset_name","artifact_data"."dataset_domain","artifact_data"."dataset_version","artifact_data"."artifact_id") IN (($1,$2,$3,$4,$5))%!!(string=123)!(string=testVersion)!(string=testDomain)!(string=testName)(EXTRA string=testProject)`).WithReply(getDBArtifactDataResponse(artifact)) GlobalMock.NewMock().WithQuery( - `SELECT * FROM "partitions" WHERE "partitions"."deleted_at" IS NULL AND (("artifact_id" IN (123))) ORDER BY partitions.created_at ASC,"partitions"."dataset_uuid" ASC`).WithReply(getDBPartitionResponse(artifact)) + `SELECT * FROM "partitions" WHERE "partitions"."artifact_id" = $1 ORDER BY partitions.created_at ASC%!(EXTRA string=123)`).WithReply(getDBPartitionResponse(artifact)) GlobalMock.NewMock().WithQuery( - `SELECT * FROM "tags" WHERE "tags"."deleted_at" IS NULL AND ((("artifact_id","dataset_uuid") IN ((123,test-uuid))))`).WithReply(getDBTagResponse(artifact)) + `SELECT * FROM "tags" WHERE ("tags"."artifact_id","tags"."dataset_uuid") IN (($1,$2))%!!(string=test-uuid)(EXTRA string=123)`).WithReply(getDBTagResponse(artifact)) getInput := models.TagKey{ DatasetProject: artifact.DatasetProject, DatasetDomain: artifact.DatasetDomain, @@ -98,13 +100,38 @@ func TestGetTag(t *testing.T) { assert.Len(t, response.Artifact.Tags, 1) } +func TestTagNotFound(t *testing.T) { + artifact := getTestArtifact() + + GlobalMock := mocket.Catcher.Reset() + GlobalMock.Logging = true + + // Only match on queries that append expected filters + GlobalMock.NewMock().WithQuery( + `SELECT * FROM "tags" WHERE "tags"."dataset_project" = $1 AND "tags"."dataset_name" = $2 AND "tags"."dataset_domain" = $3 AND "tags"."dataset_version" = $4 AND "tags"."tag_name" = $5 ORDER BY tags.created_at DESC,"tags"."created_at" LIMIT 1%!!(string=test-tag)!(string=testVersion)!(string=testDomain)!(string=testName)(EXTRA string=testProject)`).WithError( + gorm.ErrRecordNotFound, + ) + getInput := models.TagKey{ + DatasetProject: artifact.DatasetProject, + DatasetDomain: artifact.DatasetDomain, + DatasetName: artifact.DatasetName, + DatasetVersion: artifact.DatasetVersion, + TagName: "test-tag", + } + + tagRepo := NewTagRepo(utils.GetDbForTest(t), errors.NewPostgresErrorTransformer(), promutils.NewTestScope()) + _, err := tagRepo.Get(context.Background(), getInput) + assert.Error(t, err) + assert.Equal(t, "missing entity of type Tag with identifier ", err.Error()) +} + func TestTagAlreadyExists(t *testing.T) { GlobalMock := mocket.Catcher.Reset() GlobalMock.Logging = true // Only match on queries that append expected filters GlobalMock.NewMock().WithQuery( - `INSERT INTO "tags" ("created_at","updated_at","deleted_at","dataset_project","dataset_name","dataset_domain","dataset_version","tag_name","artifact_id","dataset_uuid") VALUES (?,?,?,?,?,?,?,?,?,?)`).WithError( + `INSERT INTO "tags" ("created_at","updated_at","deleted_at","dataset_project","dataset_name","dataset_domain","dataset_version","tag_name","artifact_id","dataset_uuid") VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10)`).WithError( getAlreadyExistsErr(), ) diff --git a/pkg/repositories/handle.go b/pkg/repositories/handle.go index 9c142c24..14c0039b 100644 --- a/pkg/repositories/handle.go +++ b/pkg/repositories/handle.go @@ -9,7 +9,7 @@ import ( "github.com/flyteorg/datacatalog/pkg/repositories/models" "github.com/flyteorg/flytestdlib/logger" "github.com/flyteorg/flytestdlib/promutils" - "github.com/jinzhu/gorm" + "gorm.io/gorm" ) type DBHandle struct { @@ -24,6 +24,9 @@ func NewDBHandle(dbConfigValues config.DbConfig, catalogScope promutils.Scope) ( User: dbConfigValues.User, Password: dbConfigValues.Password, ExtraOptions: dbConfigValues.ExtraOptions, + BaseConfig: config.BaseConfig{ + DisableForeignKeyConstraintWhenMigrating: true, + }, } //TODO: abstract away the type of db we are connecting to @@ -65,19 +68,34 @@ func (h *DBHandle) CreateDB(dbName string) error { return nil } -func (h *DBHandle) Migrate() { - if h.db.Dialect().GetName() == config.Postgres { - logger.Infof(context.TODO(), "Creating postgres extension uuid-ossp if it does not exist") +func (h *DBHandle) Migrate(ctx context.Context) error { + if h.db.Config.Dialector.Name() == config.Postgres { + logger.Infof(ctx, "Creating postgres extension uuid-ossp if it does not exist") h.db.Exec("CREATE EXTENSION IF NOT EXISTS \"uuid-ossp\"") } - h.db.AutoMigrate(&models.Dataset{}) - h.db.AutoMigrate(&models.Artifact{}) - h.db.AutoMigrate(&models.ArtifactData{}) - h.db.AutoMigrate(&models.Tag{}) - h.db.AutoMigrate(&models.PartitionKey{}) - h.db.AutoMigrate(&models.Partition{}) -} + if err := h.db.AutoMigrate(&models.Dataset{}); err != nil { + return err + } + + if err := h.db.Debug().AutoMigrate(&models.Artifact{}); err != nil { + return err + } -func (h *DBHandle) Close() error { - return h.db.Close() + if err := h.db.AutoMigrate(&models.ArtifactData{}); err != nil { + return err + } + + if err := h.db.AutoMigrate(&models.Tag{}); err != nil { + return err + } + + if err := h.db.AutoMigrate(&models.PartitionKey{}); err != nil { + return err + } + + if err := h.db.AutoMigrate(&models.Partition{}); err != nil { + return err + } + + return nil } diff --git a/pkg/repositories/handle_test.go b/pkg/repositories/handle_test.go index bc3641e1..3ed0cf85 100644 --- a/pkg/repositories/handle_test.go +++ b/pkg/repositories/handle_test.go @@ -17,7 +17,7 @@ func TestCreateDB(t *testing.T) { checkExists := false GlobalMock.NewMock().WithQuery( - `SELECT EXISTS(SELECT datname FROM pg_catalog.pg_database WHERE datname = testDB)`).WithCallback( + `SELECT EXISTS(SELECT datname FROM pg_catalog.pg_database WHERE datname = $1)%!(EXTRA string=testDB)`).WithCallback( func(s string, values []driver.NamedValue) { checkExists = true }, @@ -51,7 +51,7 @@ func TestDBAlreadyExists(t *testing.T) { checkExists := false GlobalMock.NewMock().WithQuery( - `SELECT EXISTS(SELECT datname FROM pg_catalog.pg_database WHERE datname = testDB)`).WithCallback( + `SELECT EXISTS(SELECT datname FROM pg_catalog.pg_database WHERE datname = $1)%!(EXTRA string=testDB)`).WithCallback( func(s string, values []driver.NamedValue) { checkExists = true }, diff --git a/pkg/repositories/models/artifact.go b/pkg/repositories/models/artifact.go index 4856c37c..4b7fa660 100644 --- a/pkg/repositories/models/artifact.go +++ b/pkg/repositories/models/artifact.go @@ -13,9 +13,9 @@ type Artifact struct { ArtifactKey DatasetUUID string `gorm:"type:uuid;index:artifacts_dataset_uuid_idx"` Dataset Dataset `gorm:"association_autocreate:false"` - ArtifactData []ArtifactData `gorm:"association_foreignkey:DatasetProject,DatasetName,DatasetDomain,DatasetVersion,ArtifactID;foreignkey:DatasetProject,DatasetName,DatasetDomain,DatasetVersion,ArtifactID"` - Partitions []Partition `gorm:"association_foreignkey:ArtifactID;foreignkey:ArtifactID"` - Tags []Tag `gorm:"association_foreignkey:ArtifactID,DatasetUUID;foreignkey:ArtifactID,DatasetUUID"` + ArtifactData []ArtifactData `gorm:"references:DatasetProject,DatasetName,DatasetDomain,DatasetVersion,ArtifactID;foreignkey:DatasetProject,DatasetName,DatasetDomain,DatasetVersion,ArtifactID"` + Partitions []Partition `gorm:"references:ArtifactID;foreignkey:ArtifactID"` + Tags []Tag `gorm:"references:ArtifactID,DatasetUUID;foreignkey:ArtifactID,DatasetUUID"` SerializedMetadata []byte } diff --git a/pkg/repositories/models/dataset.go b/pkg/repositories/models/dataset.go index d35a96b4..95fe8d43 100644 --- a/pkg/repositories/models/dataset.go +++ b/pkg/repositories/models/dataset.go @@ -12,7 +12,7 @@ type Dataset struct { BaseModel DatasetKey SerializedMetadata []byte - PartitionKeys []PartitionKey `gorm:"association_foreignkey:UUID;foreignkey:DatasetUUID"` + PartitionKeys []PartitionKey `gorm:"references:UUID;foreignkey:DatasetUUID"` } type PartitionKey struct { diff --git a/pkg/repositories/models/list.go b/pkg/repositories/models/list.go index 26307742..234c981a 100644 --- a/pkg/repositories/models/list.go +++ b/pkg/repositories/models/list.go @@ -7,9 +7,9 @@ type ListModelsInput struct { // The filters for the list ModelFilters []ModelFilter // The number of models to list - Limit uint32 + Limit int // The token to offset results by - Offset uint32 + Offset int // Parameter to sort by SortParameter SortParameter } diff --git a/pkg/repositories/models/tag.go b/pkg/repositories/models/tag.go index 057f78c3..7b239cb3 100644 --- a/pkg/repositories/models/tag.go +++ b/pkg/repositories/models/tag.go @@ -13,5 +13,5 @@ type Tag struct { TagKey ArtifactID string DatasetUUID string `gorm:"type:uuid;index:tags_dataset_uuid_idx"` - Artifact Artifact `gorm:"association_foreignkey:DatasetProject,DatasetName,DatasetDomain,DatasetVersion,ArtifactID;foreignkey:DatasetProject,DatasetName,DatasetDomain,DatasetVersion,ArtifactID"` + Artifact Artifact `gorm:"references:DatasetProject,DatasetName,DatasetDomain,DatasetVersion,ArtifactID;foreignkey:DatasetProject,DatasetName,DatasetDomain,DatasetVersion,ArtifactID"` } diff --git a/pkg/repositories/postgres_repo.go b/pkg/repositories/postgres_repo.go index 7b016f67..5da0dca7 100644 --- a/pkg/repositories/postgres_repo.go +++ b/pkg/repositories/postgres_repo.go @@ -5,7 +5,7 @@ import ( "github.com/flyteorg/datacatalog/pkg/repositories/gormimpl" "github.com/flyteorg/datacatalog/pkg/repositories/interfaces" "github.com/flyteorg/flytestdlib/promutils" - "github.com/jinzhu/gorm" + "gorm.io/gorm" ) type PostgresRepo struct { diff --git a/pkg/repositories/transformers/pagination.go b/pkg/repositories/transformers/pagination.go index 1439cb00..0965e123 100644 --- a/pkg/repositories/transformers/pagination.go +++ b/pkg/repositories/transformers/pagination.go @@ -25,13 +25,13 @@ func ApplyPagination(paginationOpts *datacatalog.PaginationOptions, input *model if len(strings.Trim(paginationOpts.Token, " ")) == 0 { offset = common.DefaultPageOffset } else { - parsedOffset, err := strconv.ParseUint(paginationOpts.Token, 10, 32) + parsedOffset, err := strconv.ParseInt(paginationOpts.Token, 10, 32) if err != nil { return errors.NewDataCatalogErrorf(codes.InvalidArgument, "Invalid token %v", offset) } - offset = uint32(parsedOffset) + offset = int(parsedOffset) } - limit = paginationOpts.Limit + limit = int(paginationOpts.Limit) sortKey = paginationOpts.SortKey sortOrder = paginationOpts.SortOrder } diff --git a/pkg/repositories/transformers/pagination_test.go b/pkg/repositories/transformers/pagination_test.go index b05a70db..7cfa3d2f 100644 --- a/pkg/repositories/transformers/pagination_test.go +++ b/pkg/repositories/transformers/pagination_test.go @@ -33,7 +33,7 @@ func TestCorrectPagination(t *testing.T) { SortOrder: datacatalog.PaginationOptions_DESCENDING, }, listModelsInput) assert.NoError(t, err) - assert.Equal(t, uint32(50), listModelsInput.Limit) - assert.Equal(t, uint32(100), listModelsInput.Offset) + assert.Equal(t, 50, listModelsInput.Limit) + assert.Equal(t, 100, listModelsInput.Offset) assert.Equal(t, "artifacts.created_at desc", listModelsInput.SortParameter.GetDBOrderExpression("artifacts")) } diff --git a/pkg/repositories/utils/test_utils.go b/pkg/repositories/utils/test_utils.go index 127ede23..87b2382b 100644 --- a/pkg/repositories/utils/test_utils.go +++ b/pkg/repositories/utils/test_utils.go @@ -6,12 +6,13 @@ import ( "testing" mocket "github.com/Selvatico/go-mocket" - "github.com/jinzhu/gorm" + "gorm.io/driver/postgres" + "gorm.io/gorm" ) func GetDbForTest(t *testing.T) *gorm.DB { mocket.Catcher.Register() - db, err := gorm.Open(mocket.DriverName, "fake args") + db, err := gorm.Open(postgres.New(postgres.Config{DriverName: mocket.DriverName})) if err != nil { t.Fatal(fmt.Sprintf("Failed to open mock db with err %v", err)) } diff --git a/pkg/runtime/application_config_provider.go b/pkg/runtime/application_config_provider.go index eef9e93b..157ffdba 100644 --- a/pkg/runtime/application_config_provider.go +++ b/pkg/runtime/application_config_provider.go @@ -40,6 +40,7 @@ func (p *ApplicationConfigurationProvider) GetDbConfig() dbconfig.DbConfig { } password = string(passwordVal) } + return dbconfig.DbConfig{ Host: dbConfigSection.Host, Port: dbConfigSection.Port, @@ -47,6 +48,7 @@ func (p *ApplicationConfigurationProvider) GetDbConfig() dbconfig.DbConfig { User: dbConfigSection.User, Password: password, ExtraOptions: dbConfigSection.ExtraOptions, + BaseConfig: dbconfig.BaseConfig{LogLevel: dbConfigSection.LogLevel}, } }