From ae589439962e1dedf2d08a8a80f8a3dbc4f22fd6 Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Wed, 6 Apr 2022 13:48:20 +0000 Subject: [PATCH] Refactors datacatalog to make every command, externally invokable (#69) --- datacalog/cmd/entrypoints/migrate.go | 71 +------ datacalog/cmd/entrypoints/serve.go | 57 +---- datacalog/cmd/entrypoints/serve_dummy.go | 38 +--- datacalog/go.mod | 34 +-- datacalog/go.sum | 127 ++++++----- datacalog/pkg/config/config.go | 3 +- datacalog/pkg/config/config_flags.go | 17 +- datacalog/pkg/config/config_flags_test.go | 34 +-- datacalog/pkg/repositories/config/database.go | 30 --- .../config/dbconfigsection_flags.go | 61 ------ .../config/dbconfigsection_flags_test.go | 200 ------------------ datacalog/pkg/repositories/config/postgres.go | 22 +- .../pkg/repositories/config/postgres_test.go | 18 +- datacalog/pkg/repositories/factory.go | 4 +- datacalog/pkg/repositories/handle.go | 30 +-- datacalog/pkg/repositories/handle_test.go | 18 ++ datacalog/pkg/repositories/initialize.go | 77 +++++++ .../pkg/rpc/datacatalogservice/service.go | 95 +++++++-- .../runtime/application_config_provider.go | 36 ++-- .../runtime/configs/data_catalog_config.go | 10 +- 20 files changed, 363 insertions(+), 619 deletions(-) delete mode 100644 datacalog/pkg/repositories/config/database.go delete mode 100755 datacalog/pkg/repositories/config/dbconfigsection_flags.go delete mode 100755 datacalog/pkg/repositories/config/dbconfigsection_flags_test.go create mode 100644 datacalog/pkg/repositories/initialize.go diff --git a/datacalog/cmd/entrypoints/migrate.go b/datacalog/cmd/entrypoints/migrate.go index 91067ce627..392e9cdd06 100644 --- a/datacalog/cmd/entrypoints/migrate.go +++ b/datacalog/cmd/entrypoints/migrate.go @@ -1,15 +1,7 @@ package entrypoints import ( - "reflect" - "github.com/flyteorg/datacatalog/pkg/repositories" - errors2 "github.com/flyteorg/datacatalog/pkg/repositories/errors" - "github.com/flyteorg/datacatalog/pkg/runtime" - "github.com/flyteorg/flytestdlib/logger" - "github.com/flyteorg/flytestdlib/promutils" - - "github.com/jackc/pgconn" "context" @@ -21,72 +13,13 @@ var parentMigrateCmd = &cobra.Command{ Short: "This command controls migration behavior for the Flyte Catalog database. Please choose a subcommand.", } -var migrationsScope = promutils.NewScope("migrations") -var migrateScope = migrationsScope.NewSubScope("migrate") - -// all postgres servers come by default with a db name named postgres -const defaultDB = "postgres" -const pqInvalidDBCode = "3D000" - // This runs all the migrations var migrateCmd = &cobra.Command{ Use: "run", Short: "This command will run all the migrations for the database", - Run: func(cmd *cobra.Command, args []string) { + RunE: func(cmd *cobra.Command, args []string) error { ctx := context.Background() - configProvider := runtime.NewConfigurationProvider() - dbConfigValues := configProvider.ApplicationConfiguration().GetDbConfig() - - dbName := dbConfigValues.DbName - dbHandle, err := repositories.NewDBHandle(dbConfigValues, migrateScope) - - if err != nil { - // if db does not exist, try creating it - cErr, ok := err.(errors2.ConnectError) - if !ok { - logger.Errorf(ctx, "Failed to cast error of type: %v, err: %v", reflect.TypeOf(err), - err) - panic(err) - } - pqError := cErr.Unwrap().(*pgconn.PgError) - if pqError.Code == pqInvalidDBCode { - logger.Warningf(ctx, "Database [%v] does not exist, trying to create it now", dbName) - - dbConfigValues.DbName = defaultDB - setupDBHandler, err := repositories.NewDBHandle(dbConfigValues, migrateScope) - if err != nil { - logger.Errorf(ctx, "Failed to connect to default DB %v, err %v", defaultDB, err) - panic(err) - } - - // Create the database if it doesn't exist - // NOTE: this is non-destructive - if for some reason one does exist an err will be thrown - err = setupDBHandler.CreateDB(dbName) - if err != nil { - logger.Errorf(ctx, "Failed to create DB %v err %v", dbName, err) - panic(err) - } - - dbConfigValues.DbName = dbName - dbHandle, err = repositories.NewDBHandle(dbConfigValues, migrateScope) - if err != nil { - logger.Errorf(ctx, "Failed to connect DB err %v", err) - panic(err) - } - } else { - logger.Errorf(ctx, "Failed to connect DB err %v", err) - panic(err) - } - } - - logger.Infof(ctx, "Created DB connection.") - - // TODO: checkpoints for migrations - if err := dbHandle.Migrate(ctx); err != nil { - logger.Errorf(ctx, "Failed to migrate. err: %v", err) - panic(err) - } - logger.Infof(ctx, "Ran DB migration successfully.") + return repositories.Migrate(ctx) }, } diff --git a/datacalog/cmd/entrypoints/serve.go b/datacalog/cmd/entrypoints/serve.go index cbfaf89fc3..2e0449729c 100644 --- a/datacalog/cmd/entrypoints/serve.go +++ b/datacalog/cmd/entrypoints/serve.go @@ -2,18 +2,14 @@ package entrypoints import ( "context" - "net" - "net/http" + + "github.com/flyteorg/flytestdlib/contextutils" + "github.com/flyteorg/flytestdlib/promutils/labeled" "github.com/flyteorg/datacatalog/pkg/config" "github.com/flyteorg/datacatalog/pkg/rpc/datacatalogservice" - "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/datacatalog" "github.com/flyteorg/flytestdlib/logger" "github.com/spf13/cobra" - "google.golang.org/grpc" - "google.golang.org/grpc/health" - "google.golang.org/grpc/health/grpc_health_v1" - "google.golang.org/grpc/reflection" ) var serveCmd = &cobra.Command{ @@ -25,56 +21,19 @@ var serveCmd = &cobra.Command{ // serve a http healthcheck endpoint go func() { - err := serveHTTPHealthcheck(ctx, cfg) + err := datacatalogservice.ServeHTTPHealthCheck(ctx, cfg) if err != nil { logger.Errorf(ctx, "Unable to serve http", config.GetConfig().GetHTTPHostAddress(), err) } }() - return serveInsecure(ctx, cfg) + // Set Keys + labeled.SetMetricKeys(contextutils.AppNameKey, contextutils.ProjectKey, contextutils.DomainKey) + + return datacatalogservice.ServeInsecure(ctx, cfg) }, } func init() { RootCmd.AddCommand(serveCmd) } - -// Create and start the gRPC server -func serveInsecure(ctx context.Context, cfg *config.Config) error { - grpcServer := newGRPCServer(ctx, cfg) - - grpcListener, err := net.Listen("tcp", cfg.GetGrpcHostAddress()) - if err != nil { - return err - } - - logger.Infof(ctx, "Serving DataCatalog Insecure on port %v", config.GetConfig().GetGrpcHostAddress()) - return grpcServer.Serve(grpcListener) -} - -// Creates a new GRPC Server with all the configuration -func newGRPCServer(_ context.Context, cfg *config.Config) *grpc.Server { - grpcServer := grpc.NewServer() - datacatalog.RegisterDataCatalogServer(grpcServer, datacatalogservice.NewDataCatalogService()) - - healthServer := health.NewServer() - healthServer.SetServingStatus("", grpc_health_v1.HealthCheckResponse_SERVING) - grpc_health_v1.RegisterHealthServer(grpcServer, healthServer) - - if cfg.GrpcServerReflection { - reflection.Register(grpcServer) - } - return grpcServer -} - -func serveHTTPHealthcheck(ctx context.Context, cfg *config.Config) error { - mux := http.NewServeMux() - - // Register Healthcheck - mux.HandleFunc("/healthcheck", func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusOK) - }) - - logger.Infof(ctx, "Serving DataCatalog http on port %v", cfg.GetHTTPHostAddress()) - return http.ListenAndServe(cfg.GetHTTPHostAddress(), mux) -} diff --git a/datacalog/cmd/entrypoints/serve_dummy.go b/datacalog/cmd/entrypoints/serve_dummy.go index 747329e3fc..a58ce0f696 100644 --- a/datacalog/cmd/entrypoints/serve_dummy.go +++ b/datacalog/cmd/entrypoints/serve_dummy.go @@ -2,15 +2,10 @@ package entrypoints import ( "context" - "net" "github.com/flyteorg/datacatalog/pkg/config" "github.com/flyteorg/datacatalog/pkg/rpc/datacatalogservice" - "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/datacatalog" - "github.com/flyteorg/flytestdlib/logger" "github.com/spf13/cobra" - "google.golang.org/grpc" - "google.golang.org/grpc/reflection" ) var serveDummyCmd = &cobra.Command{ @@ -19,41 +14,10 @@ var serveDummyCmd = &cobra.Command{ RunE: func(cmd *cobra.Command, args []string) error { ctx := context.Background() cfg := config.GetConfig() - return serveDummy(ctx, cfg) + return datacatalogservice.Serve(ctx, cfg) }, } func init() { RootCmd.AddCommand(serveDummyCmd) } - -// Create and start the gRPC server and http healthcheck endpoint -func serveDummy(ctx context.Context, cfg *config.Config) error { - // serve a http healthcheck endpoint - go func() { - err := serveHTTPHealthcheck(ctx, cfg) - if err != nil { - logger.Errorf(ctx, "Unable to serve http", cfg.GetGrpcHostAddress(), err) - } - }() - - grpcServer := newGRPCDummyServer(ctx, cfg) - - grpcListener, err := net.Listen("tcp", cfg.GetGrpcHostAddress()) - if err != nil { - return err - } - - logger.Infof(ctx, "Serving DataCatalog Insecure on port %v", cfg.GetGrpcHostAddress()) - return grpcServer.Serve(grpcListener) -} - -// Creates a new GRPC Server with all the configuration -func newGRPCDummyServer(_ context.Context, cfg *config.Config) *grpc.Server { - grpcServer := grpc.NewServer() - datacatalog.RegisterDataCatalogServer(grpcServer, &datacatalogservice.DataCatalogService{}) - if cfg.GrpcServerReflection { - reflection.Register(grpcServer) - } - return grpcServer -} diff --git a/datacalog/go.mod b/datacalog/go.mod index 4ea00a9964..5aec22a340 100644 --- a/datacalog/go.mod +++ b/datacalog/go.mod @@ -5,24 +5,28 @@ go 1.17 require ( github.com/Selvatico/go-mocket v1.0.7 github.com/flyteorg/flyteidl v0.22.1 - github.com/flyteorg/flytestdlib v0.4.7 + github.com/flyteorg/flytestdlib v0.4.17 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b github.com/golang/protobuf v1.4.3 - github.com/jackc/pgconn v1.8.1 + github.com/jackc/pgconn v1.10.1 github.com/mitchellh/mapstructure v1.4.1 github.com/satori/go.uuid v1.2.0 github.com/spf13/cobra v1.1.1 github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.7.0 google.golang.org/grpc v1.36.0 - gorm.io/driver/postgres v1.1.0 - gorm.io/gorm v1.21.9 + gorm.io/driver/postgres v1.2.3 + gorm.io/driver/sqlite v1.1.1 + gorm.io/gorm v1.22.4 ) require ( cloud.google.com/go v0.75.0 // indirect cloud.google.com/go/storage v1.12.0 // indirect - github.com/Azure/azure-sdk-for-go v51.0.0+incompatible // indirect + github.com/Azure/azure-sdk-for-go v62.3.0+incompatible // indirect + github.com/Azure/azure-sdk-for-go/sdk/azcore v0.21.1 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.3 // indirect + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.3.0 // indirect github.com/Azure/go-autorest v14.2.0+incompatible // indirect github.com/Azure/go-autorest/autorest v0.11.17 // indirect github.com/Azure/go-autorest/autorest/adal v0.9.10 // indirect @@ -36,29 +40,31 @@ require ( github.com/coocood/freecache v1.1.1 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/fatih/color v1.10.0 // indirect + github.com/flyteorg/stow v0.3.1 // indirect github.com/form3tech-oss/jwt-go v3.2.2+incompatible // indirect github.com/fsnotify/fsnotify v1.4.9 // indirect github.com/ghodss/yaml v1.0.0 // indirect github.com/go-logr/logr v0.4.0 // indirect + github.com/gofrs/uuid v4.2.0+incompatible // indirect github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect github.com/googleapis/gax-go/v2 v2.0.5 // indirect - github.com/graymeta/stow v0.2.7 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/jackc/chunkreader/v2 v2.0.1 // indirect github.com/jackc/pgio v1.0.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect - github.com/jackc/pgproto3/v2 v2.0.6 // indirect + github.com/jackc/pgproto3/v2 v2.2.0 // indirect github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect - github.com/jackc/pgtype v1.7.0 // indirect - github.com/jackc/pgx/v4 v4.11.0 // indirect + github.com/jackc/pgtype v1.9.0 // indirect + github.com/jackc/pgx/v4 v4.14.0 // indirect github.com/jinzhu/inflection v1.0.0 // indirect - github.com/jinzhu/now v1.1.2 // indirect + github.com/jinzhu/now v1.1.4 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/jstemmer/go-junit-report v0.9.1 // indirect github.com/magiconair/properties v1.8.4 // indirect github.com/mattn/go-colorable v0.1.8 // indirect github.com/mattn/go-isatty v0.0.12 // indirect + github.com/mattn/go-sqlite3 v1.14.0 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect github.com/ncw/swift v1.0.53 // indirect github.com/pelletier/go-toml v1.8.1 // indirect @@ -76,13 +82,13 @@ require ( github.com/stretchr/objx v0.3.0 // indirect github.com/subosito/gotenv v1.2.0 // indirect go.opencensus.io v0.22.6 // indirect - golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 // indirect + golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 // indirect golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5 // indirect golang.org/x/mod v0.4.1 // indirect - golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 // indirect + golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d // indirect golang.org/x/oauth2 v0.0.0-20210126194326-f9ce19ea3013 // indirect - golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c // indirect - golang.org/x/text v0.3.5 // indirect + golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 // indirect + golang.org/x/text v0.3.7 // indirect golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 // indirect golang.org/x/tools v0.1.0 // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect diff --git a/datacalog/go.sum b/datacalog/go.sum index 6831212746..ddd9aae5cd 100644 --- a/datacalog/go.sum +++ b/datacalog/go.sum @@ -40,8 +40,15 @@ cloud.google.com/go/storage v1.12.0 h1:4y3gHptW1EHVtcPAVE0eBBlFuGqEejTTG3KdIE0lU cloud.google.com/go/storage v1.12.0/go.mod h1:fFLk2dp2oAhDz8QFKwqrjdJvxSp/W2g7nillojlL5Ho= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Azure/azure-sdk-for-go v32.5.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= -github.com/Azure/azure-sdk-for-go v51.0.0+incompatible h1:p7blnyJSjJqf5jflHbSGhIhEpXIgIFmYZNg5uwqweso= github.com/Azure/azure-sdk-for-go v51.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-sdk-for-go v62.3.0+incompatible h1:Ctfsn9UoA/BB4HMYQlbPPgNXdX0tZ4tmb85+KFb2+RE= +github.com/Azure/azure-sdk-for-go v62.3.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-sdk-for-go/sdk/azcore v0.21.1 h1:qoVeMsc9/fh/yhxVaA0obYjVH/oI/ihrOoMwsLS9KSA= +github.com/Azure/azure-sdk-for-go/sdk/azcore v0.21.1/go.mod h1:fBF9PQNqB8scdgpZ3ufzaLntG0AG7C1WjPMsiFOmfHM= +github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.3 h1:E+m3SkZCN0Bf5q7YdTs5lSm2CYY3CK4spn5OmUIiQtk= +github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.3/go.mod h1:KLF4gFr6DcKFZwSuH8w8yEK6DpFl3LP5rhdvAb7Yz5I= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.3.0 h1:Px2UA+2RvSSvv+RvJNuUB6n7rs5Wsel4dXLe90Um2n4= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.3.0/go.mod h1:tPaiy8S5bQ+S5sOiDlINkp7+Ef339+Nz5L5XO+cnOHo= github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= @@ -75,6 +82,7 @@ github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0 github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/Selvatico/go-mocket v1.0.7 h1:sXuFMnMfVL9b/Os8rGXPgbOFbr4HJm8aHsulD/uMTUk= @@ -88,6 +96,7 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= +github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= @@ -149,8 +158,9 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= -github.com/dnaeon/go-vcr v1.1.0 h1:ReYa/UBrRyQdant9B4fNHGoCNKw6qh6P0fsdGmZpR7c= github.com/dnaeon/go-vcr v1.1.0/go.mod h1:M7tiix8f0r6mKKJ3Yq/kqU1OYf3MnfmBWVbPx/yU9ko= +github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= +github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= @@ -173,13 +183,13 @@ github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv github.com/fatih/color v1.10.0 h1:s36xzo75JdqLaaWoiEHk767eHiwo0598uUxyfiPkDsg= github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= github.com/fatih/structtag v1.2.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4/aAZl94= -github.com/flyteorg/flyteidl v0.21.24 h1:e2wPBK4aiLE+fw2zmhUDNg39QoJk6Lf5lQRvj8XgtFk= -github.com/flyteorg/flyteidl v0.21.24/go.mod h1:576W2ViEyjTpT+kEVHAGbrTP3HARNUZ/eCwrNPmdx9U= github.com/flyteorg/flyteidl v0.22.1 h1:9OYtiUIDTKsnNRoVGFcvUrIRbD3dxUJYgRTDnNnMRbw= github.com/flyteorg/flyteidl v0.22.1/go.mod h1:576W2ViEyjTpT+kEVHAGbrTP3HARNUZ/eCwrNPmdx9U= github.com/flyteorg/flytestdlib v0.3.13/go.mod h1:Tz8JCECAbX6VWGwFT6cmEQ+RJpZ/6L9pswu3fzWs220= -github.com/flyteorg/flytestdlib v0.4.7 h1:SMPPXI3j/MjP7D2fqaR+lPQkTrqYS7xZbwsgJI2F8SU= -github.com/flyteorg/flytestdlib v0.4.7/go.mod h1:fv1ar34LJLMTaf0tbfetisLykUlARi7rP+NQTUn6QQs= +github.com/flyteorg/flytestdlib v0.4.17 h1:C/Q0vcQNlEdX1p0TZyOZOqjSB30Sd5gqBQsuMGUQqmA= +github.com/flyteorg/flytestdlib v0.4.17/go.mod h1:GsZJqXrtgSIb5D3i75+Ud+1ufdPVEwvQegNdo22DPMo= +github.com/flyteorg/stow v0.3.1 h1:cBMbWl03Gsy5KoA5mutUYTuYpqtT7Pb8+ANGCLnmFEs= +github.com/flyteorg/stow v0.3.1/go.mod h1:HBld7ud0i4khMHwJjkO8v+NSP7ddKa/ruhf4I8fliaA= github.com/form3tech-oss/jwt-go v3.2.2+incompatible h1:TcekIExNqud5crz4xD2pavyTgWiPvpYe4Xau31I0PRk= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= @@ -196,6 +206,7 @@ github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.10.0/go.mod h1:xUsJbQ/Fp4kEt7AFgCuvyX4a71u8h9jB8tj/ORgOZ7o= +github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= @@ -213,8 +224,9 @@ github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-test/deep v1.0.7/go.mod h1:QV8Hv/iy04NyLBxAdO9njL0iVPN1S4d/A3NVv1V36o8= -github.com/gofrs/uuid v3.2.0+incompatible h1:y12jRkkFxsd7GpqdSZ+/KCs/fJbqpEXSGd4+jfEaewE= -github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/gofrs/uuid v4.2.0+incompatible h1:yyYWMnhkhrKwwr8gAOcOCYxOOscHgDS9yZgBrnJfGa0= +github.com/gofrs/uuid v4.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= @@ -299,7 +311,6 @@ github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2z github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/graymeta/stow v0.2.7 h1:b31cB1Ylw/388sYSZxnmpjT2QxC21AaQ8fRnUtE13b4= github.com/graymeta/stow v0.2.7/go.mod h1:JAs139Zr29qfsecy7b+h9DRsWXbFbsd7LCrbCDYI84k= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= @@ -350,15 +361,17 @@ github.com/jackc/chunkreader/v2 v2.0.1/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgO github.com/jackc/pgconn v0.0.0-20190420214824-7e0022ef6ba3/go.mod h1:jkELnwuX+w9qN5YIfX0fl88Ehu4XC3keFuOJJk9pcnA= github.com/jackc/pgconn v0.0.0-20190824142844-760dd75542eb/go.mod h1:lLjNuW/+OfW9/pnVKPazfWOgNfH2aPem8YQ7ilXGvJE= github.com/jackc/pgconn v0.0.0-20190831204454-2fabfa3c18b7/go.mod h1:ZJKsE/KZfsUgOEh9hBm+xYTstcNHg7UPMVJqRfQxq4s= -github.com/jackc/pgconn v1.4.0/go.mod h1:Y2O3ZDF0q4mMacyWV3AstPJpeHXWGEetiFttmq5lahk= -github.com/jackc/pgconn v1.5.0/go.mod h1:QeD3lBfpTFe8WUnPZWN5KY/mB8FGMIYRdd8P8Jr0fAI= -github.com/jackc/pgconn v1.5.1-0.20200601181101-fa742c524853/go.mod h1:QeD3lBfpTFe8WUnPZWN5KY/mB8FGMIYRdd8P8Jr0fAI= -github.com/jackc/pgconn v1.8.1 h1:ySBX7Q87vOMqKU2bbmKbUvtYhauDFclYbNDYIE1/h6s= -github.com/jackc/pgconn v1.8.1/go.mod h1:JV6m6b6jhjdmzchES0drzCcYcAHS1OPD5xu3OZ/lE2g= +github.com/jackc/pgconn v1.8.0/go.mod h1:1C2Pb36bGIP9QHGBYCjnyhqu7Rv3sGshaQUvmfGIB/o= +github.com/jackc/pgconn v1.9.0/go.mod h1:YctiPyvzfU11JFxoXokUOOKQXQmDMoJL9vJzHH8/2JY= +github.com/jackc/pgconn v1.9.1-0.20210724152538-d89c8390a530/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= +github.com/jackc/pgconn v1.10.1 h1:DzdIHIjG1AxGwoEEqS+mGsURyjt4enSmqzACXvVzOT8= +github.com/jackc/pgconn v1.10.1/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= -github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2 h1:JVX6jT/XfzNqIjye4717ITLaNwV9mWbJx0dLCpcRzdA= github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE= +github.com/jackc/pgmock v0.0.0-20201204152224-4fe30f7445fd/go.mod h1:hrBW0Enj2AZTNpt/7Y5rr2xe/9Mn757Wtb2xeBzPv2c= +github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65 h1:DadwsjnMwFjfWc9y5Wi/+Zz7xoE5ALHsRQlOctkOiHc= +github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65/go.mod h1:5R2h2EEX+qri8jOWMbJCtaPWkrrNc7OHwsp2TCqp7ak= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgproto3 v1.1.0 h1:FYYE4yRw+AgI8wXIinMlNjBbp/UitDJwfj5LqqewP1A= @@ -367,37 +380,35 @@ github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190420180111-c116219b62db/go.mod github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190609003834-432c2951c711/go.mod h1:uH0AWtUmuShn0bcesswc4aBTWGvw0cAxIJp+6OB//Wg= github.com/jackc/pgproto3/v2 v2.0.0-rc3/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= github.com/jackc/pgproto3/v2 v2.0.0-rc3.0.20190831210041-4c03ce451f29/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= -github.com/jackc/pgproto3/v2 v2.0.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= -github.com/jackc/pgproto3/v2 v2.0.6 h1:b1105ZGEMFe7aCvrT1Cca3VoVb4ZFMaFJLJcg/3zD+8= github.com/jackc/pgproto3/v2 v2.0.6/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= -github.com/jackc/pgservicefile v0.0.0-20200307190119-3430c5407db8/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= +github.com/jackc/pgproto3/v2 v2.1.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgproto3/v2 v2.2.0 h1:r7JypeP2D3onoQTCxWdTpCtJ4D+qpKr0TxvoyMhZ5ns= +github.com/jackc/pgproto3/v2 v2.2.0/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b h1:C8S2+VttkHFdOOCXJe+YGfa4vHYwlt4Zx+IVXQ97jYg= github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01CGwFsrv11mJRHWJ6aifDLfdV3aVjFF0zg= github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc= github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw= -github.com/jackc/pgtype v1.2.0/go.mod h1:5m2OfMh1wTK7x+Fk952IDmI4nw3nPrvtQdM0ZT4WpC0= -github.com/jackc/pgtype v1.3.1-0.20200510190516-8cd94a14c75a/go.mod h1:vaogEUkALtxZMCH411K+tKzNpwzCKU+AnPzBKZ+I+Po= -github.com/jackc/pgtype v1.3.1-0.20200606141011-f6355165a91c/go.mod h1:cvk9Bgu/VzJ9/lxTO5R5sf80p0DiucVtN7ZxvaC4GmQ= -github.com/jackc/pgtype v1.7.0 h1:6f4kVsW01QftE38ufBYxKciO6gyioXSC0ABIRLcZrGs= -github.com/jackc/pgtype v1.7.0/go.mod h1:ZnHF+rMePVqDKaOfJVI4Q8IVvAQMryDlDkZnKOI75BE= +github.com/jackc/pgtype v1.8.1-0.20210724151600-32e20a603178/go.mod h1:C516IlIV9NKqfsMCXTdChteoXmwgUceqaLfjg2e3NlM= +github.com/jackc/pgtype v1.9.0 h1:/SH1RxEtltvJgsDqp3TbiTFApD3mey3iygpuEGeuBXk= +github.com/jackc/pgtype v1.9.0/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4= github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y= github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM= github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc= -github.com/jackc/pgx/v4 v4.5.0/go.mod h1:EpAKPLdnTorwmPUUsqrPxy5fphV18j9q3wrfRXgo+kA= -github.com/jackc/pgx/v4 v4.6.1-0.20200510190926-94ba730bb1e9/go.mod h1:t3/cdRQl6fOLDxqtlyhe9UWgfIi9R8+8v8GKV5TRA/o= -github.com/jackc/pgx/v4 v4.6.1-0.20200606145419-4e5062306904/go.mod h1:ZDaNWkt9sW1JMiNn0kdYBaLelIhw7Pg4qd+Vk6tw7Hg= -github.com/jackc/pgx/v4 v4.11.0 h1:J86tSWd3Y7nKjwT/43xZBvpi04keQWx8gNC2YkdJhZI= -github.com/jackc/pgx/v4 v4.11.0/go.mod h1:i62xJgdrtVDsnL3U8ekyrQXEwGNTRoG7/8r+CIdYfcc= +github.com/jackc/pgx/v4 v4.12.1-0.20210724153913-640aa07df17c/go.mod h1:1QD0+tgSXP7iUjYm9C1NxKhny7lq6ee99u/z+IHFcgs= +github.com/jackc/pgx/v4 v4.14.0 h1:TgdrmgnM7VY72EuSQzBbBd4JA1RLqJolrw9nQVZABVc= +github.com/jackc/pgx/v4 v4.14.0/go.mod h1:jT3ibf/A0ZVCp89rtCIN0zCJxcE74ypROmHEZYsG/j8= github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= -github.com/jackc/puddle v1.1.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= -github.com/jackc/puddle v1.1.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.2.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= -github.com/jinzhu/now v1.1.2 h1:eVKgfIdy9b6zbWBMgFpfDPoAMifwSZagU9HmEU6zgiI= +github.com/jinzhu/now v1.1.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jinzhu/now v1.1.2/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/jinzhu/now v1.1.3/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/jinzhu/now v1.1.4 h1:tHnRBy1i5F2Dh8BAFxqFzxKqqvezXrL2OW1TnX+Mlas= +github.com/jinzhu/now v1.1.4/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= @@ -426,8 +437,9 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxv github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= @@ -436,8 +448,8 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/lib/pq v1.3.0 h1:/qkRGz8zljWiDcFvgpwUpwIAPu3r07TDvs3Rws+o/pU= -github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.10.2 h1:AqzbZs4ZoCBp+GtejcpCpcxM3zlSMx29dXbUSeVtJb8= +github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= @@ -448,7 +460,6 @@ github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= -github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8= github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= @@ -456,11 +467,11 @@ github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNx github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/mattn/go-sqlite3 v1.14.0 h1:mLyGNKR8+Vv9CAU7PphKa2hkEqxxhn8i32J6FPj1/QA= +github.com/mattn/go-sqlite3 v1.14.0/go.mod h1:JIl7NbARA7phWnGvh0LKTyg7S9BA+6gx71ShQilpsus= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= @@ -583,8 +594,8 @@ github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= -github.com/shopspring/decimal v0.0.0-20200227202807-02e2044944cc h1:jUIKcSPO9MoMJBbEoyE/RJoE8vz7Mb8AjvifMMwSyvY= -github.com/shopspring/decimal v0.0.0-20200227202807-02e2044944cc/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= +github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= +github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= @@ -681,14 +692,15 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= -golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 h1:It14KIkyBFYkHkwZ7k45minvA9aorojkyjGk9KJ5B/w= -golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 h1:7I4JAnoQBe7ZtJcBaYHi5UtiO8tQHbUSXxL+pnGRANg= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -724,6 +736,7 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1 h1:Kvvh58BN8Y9/lBi7hTekvtMpm07eUZ0ck5pRHpsMWrY= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -760,14 +773,17 @@ golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200904194848-62affa334b73/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201010224723-4f7140c49acb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 h1:qWPm9rbaAMKs8Bq/9LRpbMqxWRVUAQwMI9fVrssnTfw= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d h1:20cMwl2fHAzkJMEA+8J4JgqBQcQGzbisXo31MIeenXI= +golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -842,8 +858,10 @@ golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201214210602-f9fddec55a1e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c h1:VwygUrnw9jn88c4u8GD3rZQbqrP/tgas88tPUbBxQrk= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 h1:SrN+KX8Art/Sf4HNj6Zcz06G7VEz+7w9tdXTPOZ7+l4= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -852,8 +870,10 @@ golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.5 h1:i6eZZ+zk0SOf0xgBpEpPD18qWcJda6q1sxt3S0kzyUQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1045,8 +1065,9 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= @@ -1073,10 +1094,14 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gorm.io/driver/postgres v1.1.0 h1:afBljg7PtJ5lA6YUWluV2+xovIPhS+YiInuL3kUjrbk= -gorm.io/driver/postgres v1.1.0/go.mod h1:hXQIwafeRjJvUm+OMxcFWyswJ/vevcpPLlGocwAwuqw= -gorm.io/gorm v1.21.9 h1:INieZtn4P2Pw6xPJ8MzT0G4WUOsHq3RhfuDF1M6GW0E= -gorm.io/gorm v1.21.9/go.mod h1:F+OptMscr0P2F2qU97WT1WimdH9GaQPoDW7AYd5i2Y0= +gorm.io/driver/postgres v1.2.3 h1:f4t0TmNMy9gh3TU2PX+EppoA6YsgFnyq8Ojtddb42To= +gorm.io/driver/postgres v1.2.3/go.mod h1:pJV6RgYQPG47aM1f0QeOzFH9HxQc8JcmAgjRCgS0wjs= +gorm.io/driver/sqlite v1.1.1 h1:qtWqNAEUyi7gYSUAJXeiAMz0lUOdakZF5ia9Fqnp5G4= +gorm.io/driver/sqlite v1.1.1/go.mod h1:hm2olEcl8Tmsc6eZyxYSeznnsDaMqamBvEXLNtBg4cI= +gorm.io/gorm v1.9.19/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= +gorm.io/gorm v1.22.3/go.mod h1:F+OptMscr0P2F2qU97WT1WimdH9GaQPoDW7AYd5i2Y0= +gorm.io/gorm v1.22.4 h1:8aPcyEJhY0MAt8aY6Dc524Pn+pO29K+ydu+e/cXSpQM= +gorm.io/gorm v1.22.4/go.mod h1:1aeVC+pe9ZmvKZban/gW4QPra7PRoTEssyc922qCAkk= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/datacalog/pkg/config/config.go b/datacalog/pkg/config/config.go index b02f258930..783822c7a7 100644 --- a/datacalog/pkg/config/config.go +++ b/datacalog/pkg/config/config.go @@ -17,7 +17,8 @@ type Config struct { Secure bool `json:"secure" pflag:",Whether to run Catalog in secure mode or not"` } -var applicationConfig = config.MustRegisterSection(SectionKey, &Config{}) +var defaultConfig = &Config{GrpcPort: 8081, HTTPPort: 8080, GrpcServerReflection: true} +var applicationConfig = config.MustRegisterSection(SectionKey, defaultConfig) func GetConfig() *Config { return applicationConfig.GetConfig().(*Config) diff --git a/datacalog/pkg/config/config_flags.go b/datacalog/pkg/config/config_flags.go index 7180c88913..d05242c0d8 100755 --- a/datacalog/pkg/config/config_flags.go +++ b/datacalog/pkg/config/config_flags.go @@ -28,6 +28,15 @@ func (Config) elemValueOrNil(v interface{}) interface{} { return v } +func (Config) mustJsonMarshal(v interface{}) string { + raw, err := json.Marshal(v) + if err != nil { + panic(err) + } + + return string(raw) +} + func (Config) mustMarshalJSON(v json.Marshaler) string { raw, err := v.MarshalJSON() if err != nil { @@ -41,9 +50,9 @@ func (Config) mustMarshalJSON(v json.Marshaler) string { // flags is json-name.json-sub-name... etc. func (cfg Config) GetPFlagSet(prefix string) *pflag.FlagSet { cmdFlags := pflag.NewFlagSet("Config", pflag.ExitOnError) - cmdFlags.Int(fmt.Sprintf("%v%v", prefix, "grpcPort"), *new(int), "On which grpc port to serve Catalog") - cmdFlags.Bool(fmt.Sprintf("%v%v", prefix, "grpcServerReflection"), *new(bool), "Enable GRPC Server Reflection") - cmdFlags.Int(fmt.Sprintf("%v%v", prefix, "httpPort"), *new(int), "On which http port to serve Catalog") - cmdFlags.Bool(fmt.Sprintf("%v%v", prefix, "secure"), *new(bool), "Whether to run Catalog in secure mode or not") + cmdFlags.Int(fmt.Sprintf("%v%v", prefix, "grpcPort"), defaultConfig.GrpcPort, "On which grpc port to serve Catalog") + cmdFlags.Bool(fmt.Sprintf("%v%v", prefix, "grpcServerReflection"), defaultConfig.GrpcServerReflection, "Enable GRPC Server Reflection") + cmdFlags.Int(fmt.Sprintf("%v%v", prefix, "httpPort"), defaultConfig.HTTPPort, "On which http port to serve Catalog") + cmdFlags.Bool(fmt.Sprintf("%v%v", prefix, "secure"), defaultConfig.Secure, "Whether to run Catalog in secure mode or not") return cmdFlags } diff --git a/datacalog/pkg/config/config_flags_test.go b/datacalog/pkg/config/config_flags_test.go index 2e71510829..06482bd55f 100755 --- a/datacalog/pkg/config/config_flags_test.go +++ b/datacalog/pkg/config/config_flags_test.go @@ -84,7 +84,7 @@ func testDecodeJson_Config(t *testing.T, val, result interface{}) { assert.NoError(t, decode_Config(val, result)) } -func testDecodeSlice_Config(t *testing.T, vStringSlice, result interface{}) { +func testDecodeRaw_Config(t *testing.T, vStringSlice, result interface{}) { assert.NoError(t, decode_Config(vStringSlice, result)) } @@ -100,14 +100,6 @@ func TestConfig_SetFlags(t *testing.T) { assert.True(t, cmdFlags.HasFlags()) t.Run("Test_grpcPort", func(t *testing.T) { - t.Run("DefaultValue", func(t *testing.T) { - // Test that default value is set properly - if vInt, err := cmdFlags.GetInt("grpcPort"); err == nil { - assert.Equal(t, int(*new(int)), vInt) - } else { - assert.FailNow(t, err.Error()) - } - }) t.Run("Override", func(t *testing.T) { testValue := "1" @@ -122,14 +114,6 @@ func TestConfig_SetFlags(t *testing.T) { }) }) t.Run("Test_grpcServerReflection", func(t *testing.T) { - t.Run("DefaultValue", func(t *testing.T) { - // Test that default value is set properly - if vBool, err := cmdFlags.GetBool("grpcServerReflection"); err == nil { - assert.Equal(t, bool(*new(bool)), vBool) - } else { - assert.FailNow(t, err.Error()) - } - }) t.Run("Override", func(t *testing.T) { testValue := "1" @@ -144,14 +128,6 @@ func TestConfig_SetFlags(t *testing.T) { }) }) t.Run("Test_httpPort", func(t *testing.T) { - t.Run("DefaultValue", func(t *testing.T) { - // Test that default value is set properly - if vInt, err := cmdFlags.GetInt("httpPort"); err == nil { - assert.Equal(t, int(*new(int)), vInt) - } else { - assert.FailNow(t, err.Error()) - } - }) t.Run("Override", func(t *testing.T) { testValue := "1" @@ -166,14 +142,6 @@ func TestConfig_SetFlags(t *testing.T) { }) }) t.Run("Test_secure", func(t *testing.T) { - t.Run("DefaultValue", func(t *testing.T) { - // Test that default value is set properly - if vBool, err := cmdFlags.GetBool("secure"); err == nil { - assert.Equal(t, bool(*new(bool)), vBool) - } else { - assert.FailNow(t, err.Error()) - } - }) t.Run("Override", func(t *testing.T) { testValue := "1" diff --git a/datacalog/pkg/repositories/config/database.go b/datacalog/pkg/repositories/config/database.go deleted file mode 100644 index 2311dc5121..0000000000 --- a/datacalog/pkg/repositories/config/database.go +++ /dev/null @@ -1,30 +0,0 @@ -package config - -import "gorm.io/gorm/logger" - -//go:generate pflags DbConfigSection - -// DbConfigSection corresponds to the database section of in the config -type DbConfigSection struct { - Host string `json:"host"` - Port int `json:"port"` - DbName string `json:"dbname"` - User string `json:"username"` - // Either Password or PasswordPath must be set. - Password string `json:"password"` - PasswordPath string `json:"passwordPath"` - // See http://gorm.io/docs/connecting_to_the_database.html for available options passed, in addition to the above. - ExtraOptions string `json:"options"` - LogLevel logger.LogLevel `json:"log_level" pflag:"-,"` -} - -// DbConfig is database config. Contains values necessary to open a database connection. -type DbConfig struct { - BaseConfig - Host string `json:"host"` - Port int `json:"port"` - DbName string `json:"dbname"` - User string `json:"user"` - Password string `json:"password"` - ExtraOptions string `json:"options"` -} diff --git a/datacalog/pkg/repositories/config/dbconfigsection_flags.go b/datacalog/pkg/repositories/config/dbconfigsection_flags.go deleted file mode 100755 index de963d40c1..0000000000 --- a/datacalog/pkg/repositories/config/dbconfigsection_flags.go +++ /dev/null @@ -1,61 +0,0 @@ -// Code generated by go generate; DO NOT EDIT. -// This file was generated by robots. - -package config - -import ( - "encoding/json" - "reflect" - - "fmt" - - "github.com/spf13/pflag" -) - -// If v is a pointer, it will get its element value or the zero value of the element type. -// If v is not a pointer, it will return it as is. -func (DbConfigSection) elemValueOrNil(v interface{}) interface{} { - if t := reflect.TypeOf(v); t.Kind() == reflect.Ptr { - if reflect.ValueOf(v).IsNil() { - return reflect.Zero(t.Elem()).Interface() - } else { - return reflect.ValueOf(v).Interface() - } - } else if v == nil { - return reflect.Zero(t).Interface() - } - - return v -} - -func (DbConfigSection) mustJsonMarshal(v interface{}) string { - raw, err := json.Marshal(v) - if err != nil { - panic(err) - } - - return string(raw) -} - -func (DbConfigSection) mustMarshalJSON(v json.Marshaler) string { - raw, err := v.MarshalJSON() - if err != nil { - panic(err) - } - - return string(raw) -} - -// GetPFlagSet will return strongly types pflags for all fields in DbConfigSection and its nested types. The format of the -// flags is json-name.json-sub-name... etc. -func (cfg DbConfigSection) GetPFlagSet(prefix string) *pflag.FlagSet { - cmdFlags := pflag.NewFlagSet("DbConfigSection", pflag.ExitOnError) - cmdFlags.String(fmt.Sprintf("%v%v", prefix, "host"), *new(string), "") - cmdFlags.Int(fmt.Sprintf("%v%v", prefix, "port"), *new(int), "") - cmdFlags.String(fmt.Sprintf("%v%v", prefix, "dbname"), *new(string), "") - cmdFlags.String(fmt.Sprintf("%v%v", prefix, "username"), *new(string), "") - cmdFlags.String(fmt.Sprintf("%v%v", prefix, "password"), *new(string), "") - cmdFlags.String(fmt.Sprintf("%v%v", prefix, "passwordPath"), *new(string), "") - cmdFlags.String(fmt.Sprintf("%v%v", prefix, "options"), *new(string), "") - return cmdFlags -} diff --git a/datacalog/pkg/repositories/config/dbconfigsection_flags_test.go b/datacalog/pkg/repositories/config/dbconfigsection_flags_test.go deleted file mode 100755 index 26c7639c3e..0000000000 --- a/datacalog/pkg/repositories/config/dbconfigsection_flags_test.go +++ /dev/null @@ -1,200 +0,0 @@ -// Code generated by go generate; DO NOT EDIT. -// This file was generated by robots. - -package config - -import ( - "encoding/json" - "fmt" - "reflect" - "strings" - "testing" - - "github.com/mitchellh/mapstructure" - "github.com/stretchr/testify/assert" -) - -var dereferencableKindsDbConfigSection = map[reflect.Kind]struct{}{ - reflect.Array: {}, reflect.Chan: {}, reflect.Map: {}, reflect.Ptr: {}, reflect.Slice: {}, -} - -// Checks if t is a kind that can be dereferenced to get its underlying type. -func canGetElementDbConfigSection(t reflect.Kind) bool { - _, exists := dereferencableKindsDbConfigSection[t] - return exists -} - -// This decoder hook tests types for json unmarshaling capability. If implemented, it uses json unmarshal to build the -// object. Otherwise, it'll just pass on the original data. -func jsonUnmarshalerHookDbConfigSection(_, to reflect.Type, data interface{}) (interface{}, error) { - unmarshalerType := reflect.TypeOf((*json.Unmarshaler)(nil)).Elem() - if to.Implements(unmarshalerType) || reflect.PtrTo(to).Implements(unmarshalerType) || - (canGetElementDbConfigSection(to.Kind()) && to.Elem().Implements(unmarshalerType)) { - - raw, err := json.Marshal(data) - if err != nil { - fmt.Printf("Failed to marshal Data: %v. Error: %v. Skipping jsonUnmarshalHook", data, err) - return data, nil - } - - res := reflect.New(to).Interface() - err = json.Unmarshal(raw, &res) - if err != nil { - fmt.Printf("Failed to umarshal Data: %v. Error: %v. Skipping jsonUnmarshalHook", data, err) - return data, nil - } - - return res, nil - } - - return data, nil -} - -func decode_DbConfigSection(input, result interface{}) error { - config := &mapstructure.DecoderConfig{ - TagName: "json", - WeaklyTypedInput: true, - Result: result, - DecodeHook: mapstructure.ComposeDecodeHookFunc( - mapstructure.StringToTimeDurationHookFunc(), - mapstructure.StringToSliceHookFunc(","), - jsonUnmarshalerHookDbConfigSection, - ), - } - - decoder, err := mapstructure.NewDecoder(config) - if err != nil { - return err - } - - return decoder.Decode(input) -} - -func join_DbConfigSection(arr interface{}, sep string) string { - listValue := reflect.ValueOf(arr) - strs := make([]string, 0, listValue.Len()) - for i := 0; i < listValue.Len(); i++ { - strs = append(strs, fmt.Sprintf("%v", listValue.Index(i))) - } - - return strings.Join(strs, sep) -} - -func testDecodeJson_DbConfigSection(t *testing.T, val, result interface{}) { - assert.NoError(t, decode_DbConfigSection(val, result)) -} - -func testDecodeRaw_DbConfigSection(t *testing.T, vStringSlice, result interface{}) { - assert.NoError(t, decode_DbConfigSection(vStringSlice, result)) -} - -func TestDbConfigSection_GetPFlagSet(t *testing.T) { - val := DbConfigSection{} - cmdFlags := val.GetPFlagSet("") - assert.True(t, cmdFlags.HasFlags()) -} - -func TestDbConfigSection_SetFlags(t *testing.T) { - actual := DbConfigSection{} - cmdFlags := actual.GetPFlagSet("") - assert.True(t, cmdFlags.HasFlags()) - - t.Run("Test_host", func(t *testing.T) { - - t.Run("Override", func(t *testing.T) { - testValue := "1" - - cmdFlags.Set("host", testValue) - if vString, err := cmdFlags.GetString("host"); err == nil { - testDecodeJson_DbConfigSection(t, fmt.Sprintf("%v", vString), &actual.Host) - - } else { - assert.FailNow(t, err.Error()) - } - }) - }) - t.Run("Test_port", func(t *testing.T) { - - t.Run("Override", func(t *testing.T) { - testValue := "1" - - cmdFlags.Set("port", testValue) - if vInt, err := cmdFlags.GetInt("port"); err == nil { - testDecodeJson_DbConfigSection(t, fmt.Sprintf("%v", vInt), &actual.Port) - - } else { - assert.FailNow(t, err.Error()) - } - }) - }) - t.Run("Test_dbname", func(t *testing.T) { - - t.Run("Override", func(t *testing.T) { - testValue := "1" - - cmdFlags.Set("dbname", testValue) - if vString, err := cmdFlags.GetString("dbname"); err == nil { - testDecodeJson_DbConfigSection(t, fmt.Sprintf("%v", vString), &actual.DbName) - - } else { - assert.FailNow(t, err.Error()) - } - }) - }) - t.Run("Test_username", func(t *testing.T) { - - t.Run("Override", func(t *testing.T) { - testValue := "1" - - cmdFlags.Set("username", testValue) - if vString, err := cmdFlags.GetString("username"); err == nil { - testDecodeJson_DbConfigSection(t, fmt.Sprintf("%v", vString), &actual.User) - - } else { - assert.FailNow(t, err.Error()) - } - }) - }) - t.Run("Test_password", func(t *testing.T) { - - t.Run("Override", func(t *testing.T) { - testValue := "1" - - cmdFlags.Set("password", testValue) - if vString, err := cmdFlags.GetString("password"); err == nil { - testDecodeJson_DbConfigSection(t, fmt.Sprintf("%v", vString), &actual.Password) - - } else { - assert.FailNow(t, err.Error()) - } - }) - }) - t.Run("Test_passwordPath", func(t *testing.T) { - - t.Run("Override", func(t *testing.T) { - testValue := "1" - - cmdFlags.Set("passwordPath", testValue) - if vString, err := cmdFlags.GetString("passwordPath"); err == nil { - testDecodeJson_DbConfigSection(t, fmt.Sprintf("%v", vString), &actual.PasswordPath) - - } else { - assert.FailNow(t, err.Error()) - } - }) - }) - t.Run("Test_options", func(t *testing.T) { - - t.Run("Override", func(t *testing.T) { - testValue := "1" - - cmdFlags.Set("options", testValue) - if vString, err := cmdFlags.GetString("options"); err == nil { - testDecodeJson_DbConfigSection(t, fmt.Sprintf("%v", vString), &actual.ExtraOptions) - - } else { - assert.FailNow(t, err.Error()) - } - }) - }) -} diff --git a/datacalog/pkg/repositories/config/postgres.go b/datacalog/pkg/repositories/config/postgres.go index f8449cca96..05796a9e80 100644 --- a/datacalog/pkg/repositories/config/postgres.go +++ b/datacalog/pkg/repositories/config/postgres.go @@ -7,18 +7,22 @@ import ( "github.com/flyteorg/flytestdlib/promutils" + "github.com/flyteorg/flytestdlib/database" "gorm.io/driver/postgres" "gorm.io/gorm" ) -const Postgres = "postgres" +const ( + Postgres = "postgres" + Sqlite = "sqlite" +) // Generic interface for providing a config necessary to open a database connection. type DbConnectionConfigProvider interface { // Returns database dialector GetDialector() gorm.Dialector - GetDBConfig() DbConfig + GetDBConfig() database.DbConfig GetDSN() string } @@ -30,12 +34,12 @@ type BaseConfig struct { // PostgreSQL implementation for DbConnectionConfigProvider. type PostgresConfigProvider struct { - config DbConfig + config database.DbConfig scope promutils.Scope } // TODO : Make the Config provider itself env based -func NewPostgresConfigProvider(config DbConfig, scope promutils.Scope) DbConnectionConfigProvider { +func NewPostgresConfigProvider(config database.DbConfig, scope promutils.Scope) DbConnectionConfigProvider { return &PostgresConfigProvider{ config: config, scope: scope, @@ -43,20 +47,20 @@ func NewPostgresConfigProvider(config DbConfig, scope promutils.Scope) DbConnect } func (p *PostgresConfigProvider) GetDSN() string { - if p.config.Password == "" { + if p.config.Postgres.Password == "" { // Switch for development return fmt.Sprintf("host=%s port=%d dbname=%s user=%s sslmode=disable", - p.config.Host, p.config.Port, p.config.DbName, p.config.User) + p.config.Postgres.Host, p.config.Postgres.Port, p.config.Postgres.DbName, p.config.Postgres.User) } return fmt.Sprintf("host=%s port=%d dbname=%s user=%s password=%s %s", - p.config.Host, p.config.Port, p.config.DbName, p.config.User, p.config.Password, p.config.ExtraOptions) + p.config.Postgres.Host, p.config.Postgres.Port, p.config.Postgres.DbName, p.config.Postgres.User, p.config.Postgres.Password, p.config.Postgres.ExtraOptions) } func (p *PostgresConfigProvider) GetDialector() gorm.Dialector { return postgres.Open(p.GetDSN()) } -func (p *PostgresConfigProvider) GetDBConfig() DbConfig { +func (p *PostgresConfigProvider) GetDBConfig() database.DbConfig { return p.config } @@ -65,7 +69,7 @@ func (p *PostgresConfigProvider) GetDBConfig() DbConfig { func OpenDbConnection(config DbConnectionConfigProvider) (*gorm.DB, error) { db, err := gorm.Open(config.GetDialector(), &gorm.Config{ Logger: logger.Default.LogMode(config.GetDBConfig().LogLevel), - DisableForeignKeyConstraintWhenMigrating: config.GetDBConfig().DisableForeignKeyConstraintWhenMigrating, + DisableForeignKeyConstraintWhenMigrating: !config.GetDBConfig().EnableForeignKeyConstraintWhenMigrating, }) if err != nil { return nil, err diff --git a/datacalog/pkg/repositories/config/postgres_test.go b/datacalog/pkg/repositories/config/postgres_test.go index 02be2cd08c..19173ddaba 100644 --- a/datacalog/pkg/repositories/config/postgres_test.go +++ b/datacalog/pkg/repositories/config/postgres_test.go @@ -5,49 +5,51 @@ import ( "gorm.io/gorm/logger" + "github.com/flyteorg/flytestdlib/database" mockScope "github.com/flyteorg/flytestdlib/promutils" "github.com/stretchr/testify/assert" ) func TestConstructGormArgs(t *testing.T) { - postgresConfigProvider := NewPostgresConfigProvider(DbConfig{ + postgresConfigProvider := NewPostgresConfigProvider(database.DbConfig{Postgres: database.PostgresConfig{ Host: "localhost", Port: 5432, DbName: "postgres", User: "postgres", ExtraOptions: "sslmode=disable", - BaseConfig: BaseConfig{ - LogLevel: 3, - DisableForeignKeyConstraintWhenMigrating: true, - }, + }, + LogLevel: 3, + EnableForeignKeyConstraintWhenMigrating: false, }, mockScope.NewTestScope()) assert.Equal(t, "host=localhost port=5432 dbname=postgres user=postgres sslmode=disable", postgresConfigProvider.GetDSN()) assert.Equal(t, logger.LogLevel(3), postgresConfigProvider.GetDBConfig().LogLevel) - assert.Equal(t, true, postgresConfigProvider.GetDBConfig().DisableForeignKeyConstraintWhenMigrating) + assert.Equal(t, false, postgresConfigProvider.GetDBConfig().EnableForeignKeyConstraintWhenMigrating) } func TestConstructGormArgsWithPassword(t *testing.T) { - postgresConfigProvider := NewPostgresConfigProvider(DbConfig{ + postgresConfigProvider := NewPostgresConfigProvider(database.DbConfig{Postgres: database.PostgresConfig{ Host: "localhost", Port: 5432, DbName: "postgres", User: "postgres", Password: "pass", ExtraOptions: "sslmode=enable", + }, }, mockScope.NewTestScope()) assert.Equal(t, "host=localhost port=5432 dbname=postgres user=postgres password=pass sslmode=enable", postgresConfigProvider.GetDSN()) } func TestConstructGormArgsWithPasswordNoExtra(t *testing.T) { - postgresConfigProvider := NewPostgresConfigProvider(DbConfig{ + postgresConfigProvider := NewPostgresConfigProvider(database.DbConfig{Postgres: database.PostgresConfig{ Host: "localhost", Port: 5432, DbName: "postgres", User: "postgres", Password: "pass", + }, }, mockScope.NewTestScope()) assert.Equal(t, "host=localhost port=5432 dbname=postgres user=postgres password=pass ", postgresConfigProvider.GetDSN()) diff --git a/datacalog/pkg/repositories/factory.go b/datacalog/pkg/repositories/factory.go index ae234a1881..e79ceaec1e 100644 --- a/datacalog/pkg/repositories/factory.go +++ b/datacalog/pkg/repositories/factory.go @@ -3,6 +3,8 @@ package repositories import ( "fmt" + "github.com/flyteorg/flytestdlib/database" + "github.com/flyteorg/datacatalog/pkg/repositories/config" "github.com/flyteorg/datacatalog/pkg/repositories/errors" "github.com/flyteorg/datacatalog/pkg/repositories/interfaces" @@ -29,7 +31,7 @@ type RepositoryInterface interface { ReservationRepo() interfaces.ReservationRepo } -func GetRepository(repoType RepoConfig, dbConfig config.DbConfig, scope promutils.Scope) RepositoryInterface { +func GetRepository(repoType RepoConfig, dbConfig database.DbConfig, scope promutils.Scope) RepositoryInterface { switch repoType { case POSTGRES: db, err := config.OpenDbConnection(config.NewPostgresConfigProvider(dbConfig, scope.NewSubScope("postgres"))) diff --git a/datacalog/pkg/repositories/handle.go b/datacalog/pkg/repositories/handle.go index bb0d8cd9a1..099cf7a989 100644 --- a/datacalog/pkg/repositories/handle.go +++ b/datacalog/pkg/repositories/handle.go @@ -3,8 +3,12 @@ package repositories import ( "context" + "gorm.io/driver/sqlite" + "fmt" + "github.com/flyteorg/flytestdlib/database" + "github.com/flyteorg/datacatalog/pkg/repositories/config" "github.com/flyteorg/datacatalog/pkg/repositories/models" "github.com/flyteorg/flytestdlib/logger" @@ -16,27 +20,25 @@ type DBHandle struct { db *gorm.DB } -func NewDBHandle(dbConfigValues config.DbConfig, catalogScope promutils.Scope) (*DBHandle, error) { - dbConfig := config.DbConfig{ - Host: dbConfigValues.Host, - Port: dbConfigValues.Port, - DbName: dbConfigValues.DbName, - User: dbConfigValues.User, - Password: dbConfigValues.Password, - ExtraOptions: dbConfigValues.ExtraOptions, - BaseConfig: config.BaseConfig{ - DisableForeignKeyConstraintWhenMigrating: true, - }, +func NewDBHandle(dbConfigValues database.DbConfig, catalogScope promutils.Scope) (*DBHandle, error) { + var gormDb *gorm.DB + var err error + + switch { + case !dbConfigValues.SQLite.IsEmpty(): + gormDb, err = gorm.Open(sqlite.Open(dbConfigValues.SQLite.File)) + case !dbConfigValues.Postgres.IsEmpty(): + gormDb, err = config.OpenDbConnection(config.NewPostgresConfigProvider(dbConfigValues, catalogScope.NewSubScope(config.Postgres))) + default: + return nil, fmt.Errorf("unrecognized database config, %v. Supported only postgres and sqlite", dbConfigValues) } - //TODO: abstract away the type of db we are connecting to - db, err := config.OpenDbConnection(config.NewPostgresConfigProvider(dbConfig, catalogScope.NewSubScope("postgres"))) if err != nil { return nil, err } out := &DBHandle{ - db: db, + db: gormDb, } return out, nil diff --git a/datacalog/pkg/repositories/handle_test.go b/datacalog/pkg/repositories/handle_test.go index 3ed0cf8581..2ddeeef13a 100644 --- a/datacalog/pkg/repositories/handle_test.go +++ b/datacalog/pkg/repositories/handle_test.go @@ -1,9 +1,12 @@ package repositories import ( + "path" "testing" mocket "github.com/Selvatico/go-mocket" + "github.com/flyteorg/datacatalog/pkg/repositories/config" + "github.com/flyteorg/flytestdlib/database" "github.com/stretchr/testify/assert" "database/sql/driver" @@ -76,3 +79,18 @@ func TestDBAlreadyExists(t *testing.T) { assert.True(t, checkExists) assert.False(t, createdDB) } + +func TestNewDBHandle(t *testing.T) { + t.Run("missing DB Config", func(t *testing.T) { + _, err := NewDBHandle(database.DbConfig{}, migrateScope) + assert.Error(t, err) + }) + + t.Run("sqlite config", func(t *testing.T) { + dbFile := path.Join(t.TempDir(), "admin.db") + dbHandle, err := NewDBHandle(database.DbConfig{SQLite: database.SQLiteConfig{File: dbFile}}, migrateScope) + assert.Nil(t, err) + assert.NotNil(t, dbHandle) + assert.Equal(t, config.Sqlite, dbHandle.db.Name()) + }) +} diff --git a/datacalog/pkg/repositories/initialize.go b/datacalog/pkg/repositories/initialize.go new file mode 100644 index 0000000000..9f0bcafcf7 --- /dev/null +++ b/datacalog/pkg/repositories/initialize.go @@ -0,0 +1,77 @@ +package repositories + +import ( + "context" + "reflect" + + errors2 "github.com/flyteorg/datacatalog/pkg/repositories/errors" + "github.com/flyteorg/datacatalog/pkg/runtime" + "github.com/flyteorg/flytestdlib/logger" + "github.com/flyteorg/flytestdlib/promutils" + "github.com/jackc/pgconn" +) + +var migrationsScope = promutils.NewScope("migrations") +var migrateScope = migrationsScope.NewSubScope("migrate") + +// all postgres servers come by default with a db name named postgres +const defaultDB = "postgres" +const pqInvalidDBCode = "3D000" + +// Migrate This command will run all the migrations for the database +func Migrate(ctx context.Context) error { + configProvider := runtime.NewConfigurationProvider() + dbConfigValues := *configProvider.ApplicationConfiguration().GetDbConfig() + + dbName := dbConfigValues.Postgres.DbName + dbHandle, err := NewDBHandle(dbConfigValues, migrateScope) + + if err != nil { + // if db does not exist, try creating it + cErr, ok := err.(errors2.ConnectError) + if !ok { + logger.Errorf(ctx, "Failed to cast error of type: %v, err: %v", reflect.TypeOf(err), + err) + panic(err) + } + pqError := cErr.Unwrap().(*pgconn.PgError) + if pqError.Code == pqInvalidDBCode { + logger.Warningf(ctx, "Database [%v] does not exist, trying to create it now", dbName) + + dbConfigValues.Postgres.DbName = defaultDB + setupDBHandler, err := NewDBHandle(dbConfigValues, migrateScope) + if err != nil { + logger.Errorf(ctx, "Failed to connect to default DB %v, err %v", defaultDB, err) + panic(err) + } + + // Create the database if it doesn't exist + // NOTE: this is non-destructive - if for some reason one does exist an err will be thrown + err = setupDBHandler.CreateDB(dbName) + if err != nil { + logger.Errorf(ctx, "Failed to create DB %v err %v", dbName, err) + panic(err) + } + + dbConfigValues.Postgres.DbName = dbName + dbHandle, err = NewDBHandle(dbConfigValues, migrateScope) + if err != nil { + logger.Errorf(ctx, "Failed to connect DB err %v", err) + panic(err) + } + } else { + logger.Errorf(ctx, "Failed to connect DB err %v", err) + panic(err) + } + } + + logger.Infof(ctx, "Created DB connection.") + + // TODO: checkpoints for migrations + if err := dbHandle.Migrate(ctx); err != nil { + logger.Errorf(ctx, "Failed to migrate. err: %v", err) + panic(err) + } + logger.Infof(ctx, "Ran DB migration successfully.") + return nil +} diff --git a/datacalog/pkg/rpc/datacatalogservice/service.go b/datacalog/pkg/rpc/datacatalogservice/service.go index 53371d9813..2b0dcf3aec 100644 --- a/datacalog/pkg/rpc/datacatalogservice/service.go +++ b/datacalog/pkg/rpc/datacatalogservice/service.go @@ -3,20 +3,26 @@ package datacatalogservice import ( "context" "fmt" + "net" + "net/http" "runtime/debug" "time" + "google.golang.org/grpc" + "google.golang.org/grpc/health" + "google.golang.org/grpc/health/grpc_health_v1" + "google.golang.org/grpc/reflection" + + "github.com/flyteorg/datacatalog/pkg/config" "github.com/flyteorg/datacatalog/pkg/manager/impl" "github.com/flyteorg/datacatalog/pkg/manager/interfaces" "github.com/flyteorg/datacatalog/pkg/repositories" - "github.com/flyteorg/datacatalog/pkg/repositories/config" "github.com/flyteorg/datacatalog/pkg/runtime" catalog "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/datacatalog" "github.com/flyteorg/flytestdlib/contextutils" "github.com/flyteorg/flytestdlib/logger" "github.com/flyteorg/flytestdlib/profutils" "github.com/flyteorg/flytestdlib/promutils" - "github.com/flyteorg/flytestdlib/promutils/labeled" "github.com/flyteorg/flytestdlib/storage" ) @@ -69,9 +75,6 @@ func NewDataCatalogService() *DataCatalogService { catalogScope := promutils.NewScope(dataCatalogConfig.MetricsScope).NewSubScope("datacatalog") ctx := contextutils.WithAppName(context.Background(), "datacatalog") - // Set Keys - labeled.SetMetricKeys(contextutils.AppNameKey, contextutils.ProjectKey, contextutils.DomainKey) - defer func() { if err := recover(); err != nil { catalogScope.MustNewCounter("initialization_panic", @@ -96,15 +99,7 @@ func NewDataCatalogService() *DataCatalogService { } dbConfigValues := configProvider.ApplicationConfiguration().GetDbConfig() - dbConfig := config.DbConfig{ - Host: dbConfigValues.Host, - Port: dbConfigValues.Port, - DbName: dbConfigValues.DbName, - User: dbConfigValues.User, - Password: dbConfigValues.Password, - ExtraOptions: dbConfigValues.ExtraOptions, - } - repos := repositories.GetRepository(repositories.POSTGRES, dbConfig, catalogScope) + repos := repositories.GetRepository(repositories.POSTGRES, *dbConfigValues, catalogScope) logger.Infof(ctx, "Created DB connection.") // Serve profiling endpoint. @@ -124,3 +119,75 @@ func NewDataCatalogService() *DataCatalogService { catalogScope.NewSubScope("reservation")), } } + +// Create and start the gRPC server +func ServeInsecure(ctx context.Context, cfg *config.Config) error { + grpcServer := newGRPCServer(ctx, cfg) + + grpcListener, err := net.Listen("tcp", cfg.GetGrpcHostAddress()) + if err != nil { + return err + } + + logger.Infof(ctx, "Serving DataCatalog Insecure on port %v", config.GetConfig().GetGrpcHostAddress()) + return grpcServer.Serve(grpcListener) +} + +// Creates a new GRPC Server with all the configuration +func newGRPCServer(_ context.Context, cfg *config.Config) *grpc.Server { + grpcServer := grpc.NewServer() + catalog.RegisterDataCatalogServer(grpcServer, NewDataCatalogService()) + + healthServer := health.NewServer() + healthServer.SetServingStatus("", grpc_health_v1.HealthCheckResponse_SERVING) + grpc_health_v1.RegisterHealthServer(grpcServer, healthServer) + + if cfg.GrpcServerReflection { + reflection.Register(grpcServer) + } + return grpcServer +} + +// ServeHTTPHealthCheck create a http healthcheck endpoint +func ServeHTTPHealthCheck(ctx context.Context, cfg *config.Config) error { + mux := http.NewServeMux() + + // Register Health check + mux.HandleFunc("/healthcheck", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + }) + + logger.Infof(ctx, "Serving DataCatalog http on port %v", cfg.GetHTTPHostAddress()) + return http.ListenAndServe(cfg.GetHTTPHostAddress(), mux) +} + +// Create and start the gRPC server and http healthcheck endpoint +func Serve(ctx context.Context, cfg *config.Config) error { + // serve a http healthcheck endpoint + go func() { + err := ServeHTTPHealthCheck(ctx, cfg) + if err != nil { + logger.Errorf(ctx, "Unable to serve http", cfg.GetGrpcHostAddress(), err) + } + }() + + grpcServer := newGRPCDummyServer(ctx, cfg) + + grpcListener, err := net.Listen("tcp", cfg.GetGrpcHostAddress()) + if err != nil { + return err + } + + logger.Infof(ctx, "Serving DataCatalog Insecure on port %v", cfg.GetGrpcHostAddress()) + return grpcServer.Serve(grpcListener) +} + +// Creates a new GRPC Server with all the configuration +func newGRPCDummyServer(_ context.Context, cfg *config.Config) *grpc.Server { + grpcServer := grpc.NewServer() + catalog.RegisterDataCatalogServer(grpcServer, &DataCatalogService{}) + if cfg.GrpcServerReflection { + reflection.Register(grpcServer) + } + return grpcServer +} diff --git a/datacalog/pkg/runtime/application_config_provider.go b/datacalog/pkg/runtime/application_config_provider.go index 4b8b86bc8f..b4f7399d37 100644 --- a/datacalog/pkg/runtime/application_config_provider.go +++ b/datacalog/pkg/runtime/application_config_provider.go @@ -6,53 +6,43 @@ import ( "os" "strings" - dbconfig "github.com/flyteorg/datacatalog/pkg/repositories/config" - "github.com/flyteorg/datacatalog/pkg/runtime/configs" "github.com/flyteorg/flytestdlib/config" + + "github.com/flyteorg/datacatalog/pkg/runtime/configs" + "github.com/flyteorg/flytestdlib/database" "github.com/flyteorg/flytestdlib/logger" ) -const database = "database" const datacatalog = "datacatalog" -var databaseConfig = config.MustRegisterSection(database, &dbconfig.DbConfigSection{}) var datacatalogConfig = config.MustRegisterSection(datacatalog, &configs.DataCatalogConfig{}) // Defines the interface to return top-level config structs necessary to start up a datacatalog application. type ApplicationConfiguration interface { - GetDbConfig() dbconfig.DbConfig + GetDbConfig() *database.DbConfig GetDataCatalogConfig() configs.DataCatalogConfig } type ApplicationConfigurationProvider struct{} -func (p *ApplicationConfigurationProvider) GetDbConfig() dbconfig.DbConfig { - dbConfigSection := databaseConfig.GetConfig().(*dbconfig.DbConfigSection) - password := dbConfigSection.Password - if len(dbConfigSection.PasswordPath) > 0 { - if _, err := os.Stat(dbConfigSection.PasswordPath); os.IsNotExist(err) { +func (p *ApplicationConfigurationProvider) GetDbConfig() *database.DbConfig { + dbConfigSection := database.GetConfig() + if len(dbConfigSection.Postgres.PasswordPath) > 0 { + if _, err := os.Stat(dbConfigSection.Postgres.PasswordPath); os.IsNotExist(err) { logger.Fatalf(context.Background(), - "missing database password at specified path [%s]", dbConfigSection.PasswordPath) + "missing database password at specified path [%s]", dbConfigSection.Postgres.PasswordPath) } - passwordVal, err := ioutil.ReadFile(dbConfigSection.PasswordPath) + passwordVal, err := ioutil.ReadFile(dbConfigSection.Postgres.PasswordPath) if err != nil { logger.Fatalf(context.Background(), "failed to read database password from path [%s] with err: %v", - dbConfigSection.PasswordPath, err) + dbConfigSection.Postgres.PasswordPath, err) } // Passwords can contain special characters as long as they are percent encoded // https://www.postgresql.org/docs/current/libpq-connect.html - password = strings.TrimSpace(string(passwordVal)) + dbConfigSection.Postgres.Password = strings.TrimSpace(string(passwordVal)) } - return dbconfig.DbConfig{ - Host: dbConfigSection.Host, - Port: dbConfigSection.Port, - DbName: dbConfigSection.DbName, - User: dbConfigSection.User, - Password: password, - ExtraOptions: dbConfigSection.ExtraOptions, - BaseConfig: dbconfig.BaseConfig{LogLevel: dbConfigSection.LogLevel}, - } + return dbConfigSection } func (p *ApplicationConfigurationProvider) GetDataCatalogConfig() configs.DataCatalogConfig { diff --git a/datacalog/pkg/runtime/configs/data_catalog_config.go b/datacalog/pkg/runtime/configs/data_catalog_config.go index 1660c79fb2..c4d08244fb 100644 --- a/datacalog/pkg/runtime/configs/data_catalog_config.go +++ b/datacalog/pkg/runtime/configs/data_catalog_config.go @@ -1,12 +1,20 @@ package configs import ( + "time" + "github.com/flyteorg/flytestdlib/config" ) //go:generate pflags DataCatalogConfig --default-var=defaultConfig -var defaultConfig = &DataCatalogConfig{} +var defaultConfig = &DataCatalogConfig{ + StoragePrefix: "metadata", + MetricsScope: "datacatalog", + ProfilerPort: 10254, + HeartbeatGracePeriodMultiplier: 3, + MaxReservationHeartbeat: config.Duration{Duration: time.Second * 10}, +} // DataCatalogConfig is the base configuration to start datacatalog type DataCatalogConfig struct {