diff --git a/flytectl/.github/workflows/build.yaml b/flytectl/.github/workflows/build.yaml index 2b70c73f27..efdceac3d7 100644 --- a/flytectl/.github/workflows/build.yaml +++ b/flytectl/.github/workflows/build.yaml @@ -12,6 +12,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@v2 + with: + fetch-depth: "0" - uses: actions/cache@v2 with: path: | diff --git a/flytectl/cmd/config/subcommand/register/files_config.go b/flytectl/cmd/config/subcommand/register/files_config.go index a384ec83d5..b43232e6c1 100644 --- a/flytectl/cmd/config/subcommand/register/files_config.go +++ b/flytectl/cmd/config/subcommand/register/files_config.go @@ -17,4 +17,5 @@ type FilesConfig struct { AssumableIamRole string `json:"assumableIamRole" pflag:", custom assumable iam auth role to register launch plans with."` K8ServiceAccount string `json:"k8ServiceAccount" pflag:", custom kubernetes service account auth role to register launch plans with."` OutputLocationPrefix string `json:"outputLocationPrefix" pflag:", custom output location prefix for offloaded types (files/schemas)."` + SourceUploadPath string `json:"sourceUploadPath" pflag:", Location for source code in storage."` } diff --git a/flytectl/cmd/config/subcommand/register/filesconfig_flags.go b/flytectl/cmd/config/subcommand/register/filesconfig_flags.go index ffcb0d7ec4..a28aead6e5 100755 --- a/flytectl/cmd/config/subcommand/register/filesconfig_flags.go +++ b/flytectl/cmd/config/subcommand/register/filesconfig_flags.go @@ -19,5 +19,6 @@ func (cfg FilesConfig) GetPFlagSet(prefix string) *pflag.FlagSet { cmdFlags.StringVarP(&DefaultFilesConfig.AssumableIamRole, fmt.Sprintf("%v%v", prefix, "assumableIamRole"), "i", DefaultFilesConfig.AssumableIamRole, " Custom assumable iam auth role to register launch plans with.") cmdFlags.StringVarP(&DefaultFilesConfig.K8ServiceAccount, fmt.Sprintf("%v%v", prefix, "k8ServiceAccount"), "k", DefaultFilesConfig.K8ServiceAccount, " custom kubernetes service account auth role to register launch plans with.") cmdFlags.StringVarP(&DefaultFilesConfig.OutputLocationPrefix, fmt.Sprintf("%v%v", prefix, "outputLocationPrefix"), "l", DefaultFilesConfig.OutputLocationPrefix, " custom output location prefix for offloaded types (files/schemas).") + cmdFlags.StringVar(&DefaultFilesConfig.SourceUploadPath, fmt.Sprintf("%v%v", prefix, "sourceUploadPath"), DefaultFilesConfig.SourceUploadPath, " Location for source code in storage.") return cmdFlags } diff --git a/flytectl/cmd/config/subcommand/register/filesconfig_flags_test.go b/flytectl/cmd/config/subcommand/register/filesconfig_flags_test.go index 56a85f9234..9b2340ca5d 100755 --- a/flytectl/cmd/config/subcommand/register/filesconfig_flags_test.go +++ b/flytectl/cmd/config/subcommand/register/filesconfig_flags_test.go @@ -183,4 +183,18 @@ func TestFilesConfig_SetFlags(t *testing.T) { } }) }) + t.Run("Test_sourceUploadPath", func(t *testing.T) { + + t.Run("Override", func(t *testing.T) { + testValue := "1" + + cmdFlags.Set("sourceUploadPath", testValue) + if vString, err := cmdFlags.GetString("sourceUploadPath"); err == nil { + testDecodeJson_FilesConfig(t, fmt.Sprintf("%v", vString), &actual.SourceUploadPath) + + } else { + assert.FailNow(t, err.Error()) + } + }) + }) } diff --git a/flytectl/cmd/config/subcommand/sandbox/config_flags.go b/flytectl/cmd/config/subcommand/sandbox/config_flags.go index 8e84ee97fe..ebecc0a5b9 100755 --- a/flytectl/cmd/config/subcommand/sandbox/config_flags.go +++ b/flytectl/cmd/config/subcommand/sandbox/config_flags.go @@ -4,52 +4,15 @@ package sandbox import ( - "encoding/json" - "reflect" - "fmt" "github.com/spf13/pflag" ) -// If v is a pointer, it will get its element value or the zero value of the element type. -// If v is not a pointer, it will return it as is. -func (Config) elemValueOrNil(v interface{}) interface{} { - if t := reflect.TypeOf(v); t.Kind() == reflect.Ptr { - if reflect.ValueOf(v).IsNil() { - return reflect.Zero(t.Elem()).Interface() - } else { - return reflect.ValueOf(v).Interface() - } - } else if v == nil { - return reflect.Zero(t).Interface() - } - - return v -} - -func (Config) mustJsonMarshal(v interface{}) string { - raw, err := json.Marshal(v) - if err != nil { - panic(err) - } - - return string(raw) -} - -func (Config) mustMarshalJSON(v json.Marshaler) string { - raw, err := v.MarshalJSON() - if err != nil { - panic(err) - } - - return string(raw) -} - // GetPFlagSet will return strongly types pflags for all fields in Config and its nested types. The format of the // flags is json-name.json-sub-name... etc. func (cfg Config) GetPFlagSet(prefix string) *pflag.FlagSet { cmdFlags := pflag.NewFlagSet("Config", pflag.ExitOnError) - cmdFlags.StringVar(&DefaultConfig.SourcesPath, fmt.Sprintf("%v%v", prefix, "sourcesPath"), DefaultConfig.SourcesPath, "Path to your source code path where flyte workflows and tasks are.") + cmdFlags.StringVar(&DefaultConfig.Source, fmt.Sprintf("%v%v", prefix, "source"), DefaultConfig.Source, " Path of your source code") return cmdFlags } diff --git a/flytectl/cmd/config/subcommand/sandbox/config_flags_test.go b/flytectl/cmd/config/subcommand/sandbox/config_flags_test.go index 9f4775e8e5..09d41aa631 100755 --- a/flytectl/cmd/config/subcommand/sandbox/config_flags_test.go +++ b/flytectl/cmd/config/subcommand/sandbox/config_flags_test.go @@ -99,14 +99,14 @@ func TestConfig_SetFlags(t *testing.T) { cmdFlags := actual.GetPFlagSet("") assert.True(t, cmdFlags.HasFlags()) - t.Run("Test_sourcesPath", func(t *testing.T) { + t.Run("Test_source", func(t *testing.T) { t.Run("Override", func(t *testing.T) { testValue := "1" - cmdFlags.Set("sourcesPath", testValue) - if vString, err := cmdFlags.GetString("sourcesPath"); err == nil { - testDecodeJson_Config(t, fmt.Sprintf("%v", vString), &actual.SourcesPath) + cmdFlags.Set("source", testValue) + if vString, err := cmdFlags.GetString("source"); err == nil { + testDecodeJson_Config(t, fmt.Sprintf("%v", vString), &actual.Source) } else { assert.FailNow(t, err.Error()) diff --git a/flytectl/cmd/config/subcommand/sandbox/sandbox_config.go b/flytectl/cmd/config/subcommand/sandbox/sandbox_config.go index 87b1ea9cb0..afe85beddf 100644 --- a/flytectl/cmd/config/subcommand/sandbox/sandbox_config.go +++ b/flytectl/cmd/config/subcommand/sandbox/sandbox_config.go @@ -5,7 +5,7 @@ var ( DefaultConfig = &Config{} ) -// Config represents the config parameters exposed for the `sandbox` command. +//Config type Config struct { - SourcesPath string `json:"sourcesPath" pflag:",Path to your source code path where flyte workflows and tasks are."` + Source string `json:"source" pflag:", Path of your source code"` } diff --git a/flytectl/cmd/register/examples.go b/flytectl/cmd/register/examples.go index 9de4cfcf86..0952571c12 100644 --- a/flytectl/cmd/register/examples.go +++ b/flytectl/cmd/register/examples.go @@ -19,19 +19,21 @@ Registers all latest flytesnacks example Usage ` +) + +var ( githubOrg = "flyteorg" githubRepository = "flytesnacks" - archive = true snackReleaseURL = "https://github.com/flyteorg/flytesnacks/releases/download/%s/flytesnacks-%s.tgz" flyteManifest = "https://github.com/flyteorg/flytesnacks/releases/download/%s/flyte_tests_manifest.json" ) func registerExamplesFunc(ctx context.Context, args []string, cmdCtx cmdCore.CommandContext) error { - flytesnacks, tag, err := getFlyteTestManifest() + flytesnacks, tag, err := getFlyteTestManifest(githubOrg, githubRepository) if err != nil { return err } - rconfig.DefaultFilesConfig.Archive = archive + rconfig.DefaultFilesConfig.Archive = true for _, v := range flytesnacks { args := []string{ fmt.Sprintf(snackReleaseURL, tag, v.Name), diff --git a/flytectl/cmd/register/examples_test.go b/flytectl/cmd/register/examples_test.go new file mode 100644 index 0000000000..1c004a74a9 --- /dev/null +++ b/flytectl/cmd/register/examples_test.go @@ -0,0 +1,26 @@ +package register + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestRegisterExamplesFunc(t *testing.T) { + setup() + registerFilesSetup() + args = []string{""} + err := registerExamplesFunc(ctx, args, cmdCtx) + assert.NotNil(t, err) +} +func TestRegisterExamplesFuncErr(t *testing.T) { + setup() + registerFilesSetup() + githubRepository = "testingsnacks" + args = []string{""} + + err := registerExamplesFunc(ctx, args, cmdCtx) + // TODO (Yuvraj) make test to success after fixing flytesnacks bug + assert.NotNil(t, err) + githubRepository = "flytesnacks" +} diff --git a/flytectl/cmd/register/files.go b/flytectl/cmd/register/files.go index a33d7eea78..d758adce98 100644 --- a/flytectl/cmd/register/files.go +++ b/flytectl/cmd/register/files.go @@ -3,7 +3,9 @@ package register import ( "context" "encoding/json" + "fmt" "os" + "path/filepath" rconfig "github.com/flyteorg/flytectl/cmd/config/subcommand/register" cmdCore "github.com/flyteorg/flytectl/cmd/core" @@ -19,7 +21,18 @@ If there are already registered entities with v1 version then the command will f :: bin/flytectl register file _pb_output/* -d development -p flytesnacks + +There is no difference between registration and fast registration, In fast registration, the input provided by the user is fast serialized proto that is generated by pyflyte. If Flytectl finds any source code in users's input then it will consider registration as fast registration. Flytectl finds input file by searching an archive file whose name starts with fast and has .tar.gz extension When the user runs pyflyte with --fast flag then pyflyte creates serialize proto and it also archive create source code archive file in the same directory. +SourceUploadPath is an optional flag. By default, flytectl will create SourceUploadPath from your storage config. In case of s3 flytectl will upload code base in s3://{{DEFINE_BUCKET_IN_STORAGE_CONFIG}}/fast/{{VERSION}}-fast{{MD5_CREATED_BY_PYFLYTE}.tar.gz}. +:: + + bin/flytectl register file _pb_output/* -d development -p flytesnacks -v v2 + +In case of fast registration, If the SourceUploadPath flag is defined then In this case flytectl will not use the default directory for uploading the source code, it will override the destination path on the registration +:: + bin/flytectl register file _pb_output/* -d development -p flytesnacks -v v2 --SourceUploadPath="s3://dummy/fast" + Using archive file.Currently supported are .tgz and .tar extension files and can be local or remote file served through http/https. Use --archive flag. @@ -33,8 +46,7 @@ Using local tgz file. bin/flytectl register files _pb_output.tgz -d development -p flytesnacks --archive -If you want to continue executing registration on other files ignoring the errors including version conflicts then pass in -the continueOnError flag. +If you want to continue executing registration on other files ignoring the errors including version conflicts then pass in the continueOnError flag. :: @@ -73,9 +85,10 @@ Override Output location prefix during registration. :: bin/flytectl register file _pb_output/* -d development -p flytesnacks --continueOnError -v v2 -l "s3://dummy/prefix" - + Usage ` + sourceCodeExtension = ".tar.gz" ) func registerFromFilesFunc(ctx context.Context, args []string, cmdCtx cmdCore.CommandContext) error { @@ -83,17 +96,42 @@ func registerFromFilesFunc(ctx context.Context, args []string, cmdCtx cmdCore.Co } func Register(ctx context.Context, args []string, cmdCtx cmdCore.CommandContext) error { - dataRefs, tmpDir, _err := getSortedFileList(ctx, args) - if _err != nil { - logger.Errorf(ctx, "error while un-archiving files in tmp dir due to %v", _err) - return _err + var _err error + var dataRefs []string + + // getSerializeOutputFiles will return you all proto and source code compress file in sorted order + dataRefs, tmpDir, err := getSerializeOutputFiles(ctx, args) + if err != nil { + logger.Errorf(ctx, "error while un-archiving files in tmp dir due to %v", err) + return err + } + logger.Infof(ctx, "Parsing file... Total(%v)", len(dataRefs)) + + // It will segregate serialize output files in valid proto,Invalid files if have any and source code(In case of fast serialize input files) + sourceCode, validProto, InvalidFiles := segregateSourceAndProtos(dataRefs) + + // If any invalid files provide in input then through an error + if len(InvalidFiles) > 0 { + return fmt.Errorf("input package have some invalid files. try to run pyflyte package again %v", InvalidFiles) + } + + // In case of fast serialize input upload source code to destination bucket + var sourceCodeName string + if len(sourceCode) > 0 { + logger.Infof(ctx, "Fast Registration detected") + _, sourceCodeName = filepath.Split(sourceCode) + if err = uploadFastRegisterArtifact(ctx, sourceCode, sourceCodeName, rconfig.DefaultFilesConfig.Version); err != nil { + return fmt.Errorf("please check your Storage Config. It failed while uploading the source code. %v", err) + } + logger.Infof(ctx, "Source code successfully uploaded %v/%v ", rconfig.DefaultFilesConfig.SourceUploadPath, sourceCodeName) } - logger.Infof(ctx, "Parsing files... Total(%v)", len(dataRefs)) - fastFail := !rconfig.DefaultFilesConfig.ContinueOnError + var registerResults []Result - for i := 0; i < len(dataRefs) && !(fastFail && _err != nil); i++ { - registerResults, _err = registerFile(ctx, dataRefs[i], registerResults, cmdCtx) + fastFail := rconfig.DefaultFilesConfig.ContinueOnError + for i := 0; i < len(validProto) && !(fastFail && _err != nil); i++ { + registerResults, _err = registerFile(ctx, validProto[i], sourceCodeName, registerResults, cmdCtx) } + payload, _ := json.Marshal(registerResults) registerPrinter := printer.Printer{} _ = registerPrinter.JSONToTable(payload, projectColumns) diff --git a/flytectl/cmd/register/files_test.go b/flytectl/cmd/register/files_test.go index b1ead898a7..c79c1259d2 100644 --- a/flytectl/cmd/register/files_test.go +++ b/flytectl/cmd/register/files_test.go @@ -3,12 +3,21 @@ package register import ( "testing" + "github.com/flyteorg/flytestdlib/contextutils" + "github.com/flyteorg/flytestdlib/promutils/labeled" + "github.com/flyteorg/flytestdlib/storage" + rconfig "github.com/flyteorg/flytectl/cmd/config/subcommand/register" + "github.com/flyteorg/flytestdlib/promutils" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" ) +const ( + s3Output = "s3://dummy/prefix" +) + func TestRegisterFromFiles(t *testing.T) { t.Run("Valid registration", func(t *testing.T) { setup() @@ -21,12 +30,107 @@ func TestRegisterFromFiles(t *testing.T) { err := registerFromFilesFunc(ctx, args, cmdCtx) assert.Nil(t, err) }) - t.Run("Invalid registration file", func(t *testing.T) { + t.Run("Valid fast registration", func(t *testing.T) { setup() + testScope := promutils.NewTestScope() + labeled.SetMetricKeys(contextutils.AppNameKey, contextutils.ProjectKey, contextutils.DomainKey) registerFilesSetup() rconfig.DefaultFilesConfig.Archive = true - args = []string{"testdata/invalid.tar"} - err := registerFromFilesFunc(ctx, args, cmdCtx) + rconfig.DefaultFilesConfig.OutputLocationPrefix = s3Output + rconfig.DefaultFilesConfig.SourceUploadPath = s3Output + mockStorage, err := storage.NewDataStore(&storage.Config{ + Type: storage.TypeMemory, + }, testScope.NewSubScope("flytectl")) + assert.Nil(t, err) + Client = mockStorage + + args = []string{"testdata/flytesnacks-core.tgz"} + mockAdminClient.OnCreateTaskMatch(mock.Anything, mock.Anything).Return(nil, nil) + mockAdminClient.OnCreateWorkflowMatch(mock.Anything, mock.Anything).Return(nil, nil) + mockAdminClient.OnCreateLaunchPlanMatch(mock.Anything, mock.Anything).Return(nil, nil) + + err = registerFromFilesFunc(ctx, args, cmdCtx) + assert.Nil(t, err) + }) + t.Run("Failed fast registration while uploading the codebase", func(t *testing.T) { + setup() + registerFilesSetup() + testScope := promutils.NewTestScope() + labeled.SetMetricKeys(contextutils.AppNameKey, contextutils.ProjectKey, contextutils.DomainKey) + rconfig.DefaultFilesConfig.Archive = true + rconfig.DefaultFilesConfig.OutputLocationPrefix = s3Output + s, err := storage.NewDataStore(&storage.Config{ + Type: storage.TypeMemory, + }, testScope.NewSubScope("flytectl")) + assert.Nil(t, err) + Client = s + args = []string{"testdata/flytesnacks-core.tgz"} + mockAdminClient.OnCreateTaskMatch(mock.Anything, mock.Anything).Return(nil, nil) + mockAdminClient.OnCreateWorkflowMatch(mock.Anything, mock.Anything).Return(nil, nil) + mockAdminClient.OnCreateLaunchPlanMatch(mock.Anything, mock.Anything).Return(nil, nil) + err = Register(ctx, args, cmdCtx) + assert.Nil(t, err) + }) + t.Run("Failed registeration because of invalid files", func(t *testing.T) { + setup() + registerFilesSetup() + testScope := promutils.NewTestScope() + labeled.SetMetricKeys(contextutils.AppNameKey, contextutils.ProjectKey, contextutils.DomainKey) + rconfig.DefaultFilesConfig.Archive = true + rconfig.DefaultFilesConfig.SourceUploadPath = "" + s, err := storage.NewDataStore(&storage.Config{ + Type: storage.TypeMemory, + }, testScope.NewSubScope("flytectl")) + Client = s + assert.Nil(t, err) + args = []string{"testdata/invalid-fast.tgz"} + mockAdminClient.OnCreateTaskMatch(mock.Anything, mock.Anything).Return(nil, nil) + mockAdminClient.OnCreateWorkflowMatch(mock.Anything, mock.Anything).Return(nil, nil) + mockAdminClient.OnCreateLaunchPlanMatch(mock.Anything, mock.Anything).Return(nil, nil) + err = registerFromFilesFunc(ctx, args, cmdCtx) assert.NotNil(t, err) }) + + t.Run("Valid registration of fast serialize", func(t *testing.T) { + setup() + registerFilesSetup() + testScope := promutils.NewTestScope() + labeled.SetMetricKeys(contextutils.AppNameKey, contextutils.ProjectKey, contextutils.DomainKey) + rconfig.DefaultFilesConfig.Archive = true + + rconfig.DefaultFilesConfig.OutputLocationPrefix = s3Output + rconfig.DefaultFilesConfig.SourceUploadPath = s3Output + s, err := storage.NewDataStore(&storage.Config{ + Type: storage.TypeMemory, + }, testScope.NewSubScope("flytectl")) + Client = s + assert.Nil(t, err) + args = []string{"testdata/flytesnacks-core.tgz"} + mockAdminClient.OnCreateTaskMatch(mock.Anything, mock.Anything).Return(nil, nil) + mockAdminClient.OnCreateWorkflowMatch(mock.Anything, mock.Anything).Return(nil, nil) + mockAdminClient.OnCreateLaunchPlanMatch(mock.Anything, mock.Anything).Return(nil, nil) + err = registerFromFilesFunc(ctx, args, cmdCtx) + assert.Nil(t, err) + }) + + t.Run("Registration with proto files ", func(t *testing.T) { + setup() + registerFilesSetup() + testScope := promutils.NewTestScope() + labeled.SetMetricKeys(contextutils.AppNameKey, contextutils.ProjectKey, contextutils.DomainKey) + rconfig.DefaultFilesConfig.Archive = false + rconfig.DefaultFilesConfig.OutputLocationPrefix = s3Output + rconfig.DefaultFilesConfig.SourceUploadPath = "" + s, err := storage.NewDataStore(&storage.Config{ + Type: storage.TypeMemory, + }, testScope.NewSubScope("flytectl")) + Client = s + assert.Nil(t, err) + args = []string{"testdata/69_core.flyte_basics.lp.greet_1.pb"} + mockAdminClient.OnCreateTaskMatch(mock.Anything, mock.Anything).Return(nil, nil) + mockAdminClient.OnCreateWorkflowMatch(mock.Anything, mock.Anything).Return(nil, nil) + mockAdminClient.OnCreateLaunchPlanMatch(mock.Anything, mock.Anything).Return(nil, nil) + err = registerFromFilesFunc(ctx, args, cmdCtx) + assert.Nil(t, err) + }) } diff --git a/flytectl/cmd/register/register_util.go b/flytectl/cmd/register/register_util.go index 8f3883a573..14dfa4be62 100644 --- a/flytectl/cmd/register/register_util.go +++ b/flytectl/cmd/register/register_util.go @@ -11,9 +11,14 @@ import ( "io/ioutil" "net/http" "os" + "path/filepath" "sort" "strings" + "github.com/flyteorg/flytestdlib/contextutils" + "github.com/flyteorg/flytestdlib/promutils" + "github.com/flyteorg/flytestdlib/promutils/labeled" + "github.com/google/go-github/github" "github.com/flyteorg/flytectl/cmd/config" @@ -31,10 +36,14 @@ import ( "google.golang.org/grpc/status" ) +// Variable define in serialized proto that needs to be replace in registration time const registrationProjectPattern = "{{ registration.project }}" const registrationDomainPattern = "{{ registration.domain }}" const registrationVersionPattern = "{{ registration.version }}" +// Additional variable define in fast serialized proto that needs to be replace in registration time +const registrationRemotePackagePattern = "{{ .remote_package_path }}" + type Result struct { Name string Status string @@ -47,6 +56,7 @@ type HTTPClient interface { } var FlyteSnacksRelease []FlyteSnack +var Client *storage.DataStore // FlyteSnack Defines flyte test manifest structure type FlyteSnack struct { @@ -200,6 +210,21 @@ func hydrateIdentifier(identifier *core.Identifier) { } } +func hydrateTaskSpec(task *admin.TaskSpec, sourceCode string) error { + if task.Template.GetContainer() != nil { + for k := range task.Template.GetContainer().Args { + if task.Template.GetContainer().Args[k] == "" || task.Template.GetContainer().Args[k] == registrationRemotePackagePattern { + remotePath, err := getRemoteStoragePath(context.Background(), Client, rconfig.DefaultFilesConfig.SourceUploadPath, sourceCode, rconfig.DefaultFilesConfig.Version) + if err != nil { + return err + } + task.Template.GetContainer().Args[k] = string(remotePath) + } + } + } + return nil +} + func hydrateLaunchPlanSpec(lpSpec *admin.LaunchPlanSpec) { assumableIamRole := len(rconfig.DefaultFilesConfig.AssumableIamRole) > 0 k8ServiceAcct := len(rconfig.DefaultFilesConfig.K8ServiceAccount) > 0 @@ -217,7 +242,7 @@ func hydrateLaunchPlanSpec(lpSpec *admin.LaunchPlanSpec) { } } -func hydrateSpec(message proto.Message) error { +func hydrateSpec(message proto.Message, sourceCode string) error { switch v := message.(type) { case *admin.LaunchPlan: launchPlan := message.(*admin.LaunchPlan) @@ -242,8 +267,13 @@ func hydrateSpec(message proto.Message) error { case *admin.TaskSpec: taskSpec := message.(*admin.TaskSpec) hydrateIdentifier(taskSpec.Template.Id) + // In case of fast serialize input proto also have on additional variable to substitute i.e destination bucket for source code + if err := hydrateTaskSpec(taskSpec, sourceCode); err != nil { + return err + } + default: - return fmt.Errorf("Unknown type %T", v) + return fmt.Errorf("unknown type %T", v) } return nil } @@ -261,29 +291,30 @@ func DownloadFileFromHTTP(ctx context.Context, ref storage.DataReference) (io.Re } /* -Get file list from the args list. -If the archive flag is on then download the archives to temp directory and extract it. +Get serialize output file list from the args list. +If the archive flag is on then download the archives to temp directory and extract it. In case of fast register it will also return the compressed source code The o/p of this function would be sorted list of the file locations. */ -func getSortedFileList(ctx context.Context, args []string) ([]string, string, error) { +func getSerializeOutputFiles(ctx context.Context, args []string) ([]string, string, error) { if !rconfig.DefaultFilesConfig.Archive { /* * Sorting is required for non-archived case since its possible for the user to pass in a list of unordered * serialized protobuf files , but flyte expects them to be registered in topologically sorted order that it had * generated otherwise the registration can fail if the dependent files are not registered earlier. */ + sort.Strings(args) return args, "", nil } + tempDir, err := ioutil.TempDir("/tmp", "register") if err != nil { return nil, tempDir, err } - dataRefs := args var unarchivedFiles []string - for i := 0; i < len(dataRefs); i++ { - dataRefReaderCloser, err := getArchiveReaderCloser(ctx, dataRefs[i]) + for _, v := range args { + dataRefReaderCloser, err := getArchiveReaderCloser(ctx, v) if err != nil { return unarchivedFiles, tempDir, err } @@ -295,6 +326,7 @@ func getSortedFileList(ctx context.Context, args []string) ([]string, string, er return unarchivedFiles, tempDir, err } } + /* * Similarly in case of archived files, it possible to have an archive created in totally different order than the * listing order of the serialized files which is required by flyte. Hence we explicitly sort here after unarchiving it. @@ -338,10 +370,11 @@ func readAndCopyArchive(src io.Reader, tempDir string, unarchivedFiles []string) } } -func registerFile(ctx context.Context, fileName string, registerResults []Result, cmdCtx cmdCore.CommandContext) ([]Result, error) { +func registerFile(ctx context.Context, fileName, sourceCode string, registerResults []Result, cmdCtx cmdCore.CommandContext) ([]Result, error) { var registerResult Result var fileContents []byte var err error + if fileContents, err = ioutil.ReadFile(fileName); err != nil { registerResults = append(registerResults, Result{Name: fileName, Status: "Failed", Info: fmt.Sprintf("Error reading file due to %v", err)}) return registerResults, err @@ -352,12 +385,15 @@ func registerFile(ctx context.Context, fileName string, registerResults []Result registerResults = append(registerResults, registerResult) return registerResults, err } - if err := hydrateSpec(spec); err != nil { + + if err := hydrateSpec(spec, sourceCode); err != nil { registerResult = Result{Name: fileName, Status: "Failed", Info: fmt.Sprintf("Error hydrating spec due to %v", err)} registerResults = append(registerResults, registerResult) return registerResults, err } + logger.Debugf(ctx, "Hydrated spec : %v", getJSONSpec(spec)) + if err := register(ctx, spec, cmdCtx); err != nil { // If error is AlreadyExists then dont consider this to be an error but just a warning state if grpcError := status.Code(err); grpcError == codes.AlreadyExists { @@ -369,6 +405,7 @@ func registerFile(ctx context.Context, fileName string, registerResults []Result registerResults = append(registerResults, registerResult) return registerResults, err } + registerResult = Result{Name: fileName, Status: "Success", Info: "Successfully registered file"} logger.Debugf(ctx, "Successfully registered %v", fileName) registerResults = append(registerResults, registerResult) @@ -383,10 +420,12 @@ func getArchiveReaderCloser(ctx context.Context, ref string) (io.ReadCloser, err if err != nil { return nil, err } + var dataRefReaderCloser io.ReadCloser + if ext != "tar" && ext != "tgz" { return nil, errors.New("only .tar and .tgz extension archives are supported") } - var dataRefReaderCloser io.ReadCloser + if scheme == "http" || scheme == "https" { dataRefReaderCloser, err = DownloadFileFromHTTP(ctx, dataRef) } else { @@ -414,13 +453,16 @@ func getJSONSpec(message proto.Message) string { return jsonSpec } -func getFlyteTestManifest() ([]FlyteSnack, string, error) { +func getFlyteTestManifest(org, repository string) ([]FlyteSnack, string, error) { c := github.NewClient(nil) opt := &github.ListOptions{Page: 1, PerPage: 1} - releases, _, err := c.Repositories.ListReleases(context.Background(), githubOrg, githubRepository, opt) + releases, _, err := c.Repositories.ListReleases(context.Background(), org, repository, opt) if err != nil { return nil, "", err } + if len(releases) == 0 { + return nil, "", fmt.Errorf("Repository doesn't have any release") + } response, err := http.Get(fmt.Sprintf(flyteManifest, *releases[0].TagName)) if err != nil { return nil, "", err @@ -437,4 +479,89 @@ func getFlyteTestManifest() ([]FlyteSnack, string, error) { return nil, "", err } return FlyteSnacksRelease, *releases[0].TagName, nil + +} + +func getRemoteStoragePath(ctx context.Context, s *storage.DataStore, remoteLocation, file, identifier string) (storage.DataReference, error) { + remotePath, err := s.ConstructReference(ctx, storage.DataReference(remoteLocation), fmt.Sprintf("%v-%v", identifier, file)) + if err != nil { + return storage.DataReference(""), err + } + return remotePath, nil +} + +func uploadFastRegisterArtifact(ctx context.Context, file, sourceCodeName, version string) error { + dataStore, err := getStorageClient(ctx) + if err != nil { + return err + } + var dataRefReaderCloser io.ReadCloser + remotePath := storage.DataReference(rconfig.DefaultFilesConfig.SourceUploadPath) + if len(rconfig.DefaultFilesConfig.SourceUploadPath) == 0 { + remotePath, err = dataStore.ConstructReference(ctx, dataStore.GetBaseContainerFQN(ctx), "fast") + if err != nil { + return err + } + } + rconfig.DefaultFilesConfig.SourceUploadPath = string(remotePath) + fullRemotePath, err := getRemoteStoragePath(ctx, dataStore, rconfig.DefaultFilesConfig.SourceUploadPath, sourceCodeName, version) + if err != nil { + return err + } + raw, err := json.Marshal(file) + if err != nil { + return err + } + dataRefReaderCloser, err = os.Open(file) + if err != nil { + return err + } + dataRefReaderCloser, err = gzip.NewReader(dataRefReaderCloser) + if err != nil { + return err + } + if err := dataStore.ComposedProtobufStore.WriteRaw(ctx, fullRemotePath, int64(len(raw)), storage.Options{}, dataRefReaderCloser); err != nil { + return err + } + return nil +} + +func getStorageClient(ctx context.Context) (*storage.DataStore, error) { + if Client != nil { + return Client, nil + } + testScope := promutils.NewTestScope() + // Set Keys + labeled.SetMetricKeys(contextutils.AppNameKey, contextutils.ProjectKey, contextutils.DomainKey) + s, err := storage.NewDataStore(storage.GetConfig(), testScope.NewSubScope("flytectl")) + if err != nil { + logger.Errorf(ctx, "error while creating storage client %v", err) + return Client, err + } + Client = s + return Client, nil +} + +func isFastRegister(file string) bool { + _, f := filepath.Split(file) + // Pyflyte always archive source code with a name that start with fast and have an extension .tar.gz + if strings.HasPrefix(f, "fast") && strings.HasSuffix(f, sourceCodeExtension) { + return true + } + return false +} + +func segregateSourceAndProtos(dataRefs []string) (string, []string, []string) { + var validProto, InvalidFiles []string + var sourceCode string + for _, v := range dataRefs { + if isFastRegister(v) { + sourceCode = v + } else if strings.HasSuffix(v, ".pb") { + validProto = append(validProto, v) + } else { + InvalidFiles = append(InvalidFiles, v) + } + } + return sourceCode, validProto, InvalidFiles } diff --git a/flytectl/cmd/register/register_util_test.go b/flytectl/cmd/register/register_util_test.go index b7d66ada8c..b4b11563a4 100644 --- a/flytectl/cmd/register/register_util_test.go +++ b/flytectl/cmd/register/register_util_test.go @@ -9,6 +9,13 @@ import ( "strings" "testing" + "github.com/flyteorg/flytestdlib/contextutils" + "github.com/flyteorg/flytestdlib/promutils" + "github.com/flyteorg/flytestdlib/promutils/labeled" + "github.com/flyteorg/flytestdlib/storage" + + "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core" + rconfig "github.com/flyteorg/flytectl/cmd/config/subcommand/register" cmdCore "github.com/flyteorg/flytectl/cmd/core" u "github.com/flyteorg/flytectl/cmd/testutils" @@ -50,24 +57,12 @@ func registerFilesSetup() { rconfig.DefaultFilesConfig.OutputLocationPrefix = "" } -func TestGetSortedFileList(t *testing.T) { - setup() - registerFilesSetup() - rconfig.DefaultFilesConfig.Archive = false - args = []string{"file2", "file1"} - fileList, tmpDir, err := getSortedFileList(ctx, args) - assert.Equal(t, "file1", fileList[0]) - assert.Equal(t, "file2", fileList[1]) - assert.Equal(t, tmpDir, "") - assert.Nil(t, err) -} - func TestGetSortedArchivedFileWithParentFolderList(t *testing.T) { setup() registerFilesSetup() rconfig.DefaultFilesConfig.Archive = true args = []string{"testdata/valid-parent-folder-register.tar"} - fileList, tmpDir, err := getSortedFileList(ctx, args) + fileList, tmpDir, err := getSerializeOutputFiles(ctx, args) assert.Equal(t, len(fileList), 4) assert.Equal(t, filepath.Join(tmpDir, "parentfolder", "014_recipes.core.basic.basic_workflow.t1_1.pb"), fileList[0]) assert.Equal(t, filepath.Join(tmpDir, "parentfolder", "015_recipes.core.basic.basic_workflow.t2_1.pb"), fileList[1]) @@ -84,7 +79,7 @@ func TestGetSortedArchivedFileList(t *testing.T) { registerFilesSetup() rconfig.DefaultFilesConfig.Archive = true args = []string{"testdata/valid-register.tar"} - fileList, tmpDir, err := getSortedFileList(ctx, args) + fileList, tmpDir, err := getSerializeOutputFiles(ctx, args) assert.Equal(t, len(fileList), 4) assert.Equal(t, filepath.Join(tmpDir, "014_recipes.core.basic.basic_workflow.t1_1.pb"), fileList[0]) assert.Equal(t, filepath.Join(tmpDir, "015_recipes.core.basic.basic_workflow.t2_1.pb"), fileList[1]) @@ -101,7 +96,7 @@ func TestGetSortedArchivedFileUnorderedList(t *testing.T) { registerFilesSetup() rconfig.DefaultFilesConfig.Archive = true args = []string{"testdata/valid-unordered-register.tar"} - fileList, tmpDir, err := getSortedFileList(ctx, args) + fileList, tmpDir, err := getSerializeOutputFiles(ctx, args) assert.Equal(t, len(fileList), 4) assert.Equal(t, filepath.Join(tmpDir, "014_recipes.core.basic.basic_workflow.t1_1.pb"), fileList[0]) assert.Equal(t, filepath.Join(tmpDir, "015_recipes.core.basic.basic_workflow.t2_1.pb"), fileList[1]) @@ -118,7 +113,7 @@ func TestGetSortedArchivedCorruptedFileList(t *testing.T) { registerFilesSetup() rconfig.DefaultFilesConfig.Archive = true args = []string{"testdata/invalid.tar"} - fileList, tmpDir, err := getSortedFileList(ctx, args) + fileList, tmpDir, err := getSerializeOutputFiles(ctx, args) assert.Equal(t, len(fileList), 0) assert.True(t, strings.HasPrefix(tmpDir, "/tmp/register")) assert.NotNil(t, err) @@ -131,7 +126,7 @@ func TestGetSortedArchivedTgzList(t *testing.T) { registerFilesSetup() rconfig.DefaultFilesConfig.Archive = true args = []string{"testdata/valid-register.tgz"} - fileList, tmpDir, err := getSortedFileList(ctx, args) + fileList, tmpDir, err := getSerializeOutputFiles(ctx, args) assert.Equal(t, len(fileList), 4) assert.Equal(t, filepath.Join(tmpDir, "014_recipes.core.basic.basic_workflow.t1_1.pb"), fileList[0]) assert.Equal(t, filepath.Join(tmpDir, "015_recipes.core.basic.basic_workflow.t2_1.pb"), fileList[1]) @@ -147,7 +142,7 @@ func TestGetSortedArchivedCorruptedTgzFileList(t *testing.T) { setup() rconfig.DefaultFilesConfig.Archive = true args = []string{"testdata/invalid.tgz"} - fileList, tmpDir, err := getSortedFileList(ctx, args) + fileList, tmpDir, err := getSerializeOutputFiles(ctx, args) assert.Equal(t, 0, len(fileList)) assert.True(t, strings.HasPrefix(tmpDir, "/tmp/register")) assert.NotNil(t, err) @@ -160,7 +155,7 @@ func TestGetSortedArchivedInvalidArchiveFileList(t *testing.T) { registerFilesSetup() rconfig.DefaultFilesConfig.Archive = true args = []string{"testdata/invalid-extension-register.zip"} - fileList, tmpDir, err := getSortedFileList(ctx, args) + fileList, tmpDir, err := getSerializeOutputFiles(ctx, args) assert.Equal(t, 0, len(fileList)) assert.True(t, strings.HasPrefix(tmpDir, "/tmp/register")) assert.NotNil(t, err) @@ -173,7 +168,7 @@ func TestGetSortedArchivedFileThroughInvalidHttpList(t *testing.T) { setup() rconfig.DefaultFilesConfig.Archive = true args = []string{"http://invalidhost:invalidport/testdata/valid-register.tar"} - fileList, tmpDir, err := getSortedFileList(ctx, args) + fileList, tmpDir, err := getSerializeOutputFiles(ctx, args) assert.Equal(t, 0, len(fileList)) assert.True(t, strings.HasPrefix(tmpDir, "/tmp/register")) assert.NotNil(t, err) @@ -186,7 +181,7 @@ func TestGetSortedArchivedFileThroughValidHttpList(t *testing.T) { registerFilesSetup() rconfig.DefaultFilesConfig.Archive = true args = []string{"http://dummyhost:80/testdata/valid-register.tar"} - fileList, tmpDir, err := getSortedFileList(ctx, args) + fileList, tmpDir, err := getSerializeOutputFiles(ctx, args) assert.Equal(t, len(fileList), 4) assert.Equal(t, filepath.Join(tmpDir, "014_recipes.core.basic.basic_workflow.t1_1.pb"), fileList[0]) assert.Equal(t, filepath.Join(tmpDir, "015_recipes.core.basic.basic_workflow.t2_1.pb"), fileList[1]) @@ -204,7 +199,7 @@ func TestGetSortedArchivedFileThroughValidHttpWithNullContextList(t *testing.T) rconfig.DefaultFilesConfig.Archive = true args = []string{"http://dummyhost:80/testdata/valid-register.tar"} ctx = nil - fileList, tmpDir, err := getSortedFileList(ctx, args) + fileList, tmpDir, err := getSerializeOutputFiles(ctx, args) assert.Equal(t, len(fileList), 0) assert.True(t, strings.HasPrefix(tmpDir, "/tmp/register")) assert.NotNil(t, err) @@ -220,7 +215,7 @@ func TestRegisterFile(t *testing.T) { mockAdminClient.OnCreateTaskMatch(mock.Anything, mock.Anything).Return(nil, nil) args = []string{"testdata/69_core.flyte_basics.lp.greet_1.pb"} var registerResults []Result - results, err := registerFile(ctx, args[0], registerResults, cmdCtx) + results, err := registerFile(ctx, args[0], "", registerResults, cmdCtx) assert.Equal(t, 1, len(results)) assert.Nil(t, err) }) @@ -229,7 +224,7 @@ func TestRegisterFile(t *testing.T) { registerFilesSetup() args = []string{"testdata/non-existent.pb"} var registerResults []Result - results, err := registerFile(ctx, args[0], registerResults, cmdCtx) + results, err := registerFile(ctx, args[0], "", registerResults, cmdCtx) assert.Equal(t, 1, len(results)) assert.Equal(t, "Failed", results[0].Status) assert.Equal(t, "Error reading file due to open testdata/non-existent.pb: no such file or directory", results[0].Info) @@ -240,7 +235,7 @@ func TestRegisterFile(t *testing.T) { registerFilesSetup() args = []string{"testdata/valid-register.tar"} var registerResults []Result - results, err := registerFile(ctx, args[0], registerResults, cmdCtx) + results, err := registerFile(ctx, args[0], "", registerResults, cmdCtx) assert.Equal(t, 1, len(results)) assert.Equal(t, "Failed", results[0].Status) assert.Equal(t, "Error unmarshalling file due to failed unmarshalling file testdata/valid-register.tar", results[0].Info) @@ -253,7 +248,7 @@ func TestRegisterFile(t *testing.T) { status.Error(codes.AlreadyExists, "AlreadyExists")) args = []string{"testdata/69_core.flyte_basics.lp.greet_1.pb"} var registerResults []Result - results, err := registerFile(ctx, args[0], registerResults, cmdCtx) + results, err := registerFile(ctx, args[0], "", registerResults, cmdCtx) assert.Equal(t, 1, len(results)) assert.Equal(t, "Success", results[0].Status) assert.Equal(t, "AlreadyExists", results[0].Info) @@ -266,7 +261,7 @@ func TestRegisterFile(t *testing.T) { status.Error(codes.InvalidArgument, "Invalid")) args = []string{"testdata/69_core.flyte_basics.lp.greet_1.pb"} var registerResults []Result - results, err := registerFile(ctx, args[0], registerResults, cmdCtx) + results, err := registerFile(ctx, args[0], "", registerResults, cmdCtx) assert.Equal(t, 1, len(results)) assert.Equal(t, "Failed", results[0].Status) assert.Equal(t, "Error registering file due to rpc error: code = InvalidArgument desc = Invalid", results[0].Info) @@ -312,9 +307,100 @@ func TestHydrateLaunchPlanSpec(t *testing.T) { } func TestFlyteManifest(t *testing.T) { - flytesnacks, tag, err := getFlyteTestManifest() + _, tag, err := getFlyteTestManifest(githubOrg, githubRepository) assert.Nil(t, err) assert.Contains(t, tag, "v") assert.NotEmpty(t, tag) - assert.Greater(t, len(flytesnacks), 1) +} + +func TestUploadFastRegisterArtifact(t *testing.T) { + t.Run("Successful upload", func(t *testing.T) { + testScope := promutils.NewTestScope() + labeled.SetMetricKeys(contextutils.AppNameKey, contextutils.ProjectKey, contextutils.DomainKey) + s, err := storage.NewDataStore(&storage.Config{ + Type: storage.TypeMemory, + }, testScope.NewSubScope("flytectl")) + assert.Nil(t, err) + Client = s + err = uploadFastRegisterArtifact(ctx, "testdata/flytesnacks-core.tgz", "flytesnacks-core.tgz", "") + assert.Nil(t, err) + }) + t.Run("Failed upload", func(t *testing.T) { + testScope := promutils.NewTestScope() + labeled.SetMetricKeys(contextutils.AppNameKey, contextutils.ProjectKey, contextutils.DomainKey) + s, err := storage.NewDataStore(&storage.Config{ + Type: storage.TypeMemory, + }, testScope.NewSubScope("flytectl")) + assert.Nil(t, err) + Client = s + err = uploadFastRegisterArtifact(ctx, "testdata/flytesnacks-core.tgz", "", "") + assert.Nil(t, err) + }) + t.Run("Failed upload", func(t *testing.T) { + testScope := promutils.NewTestScope() + labeled.SetMetricKeys(contextutils.AppNameKey, contextutils.ProjectKey, contextutils.DomainKey) + s, err := storage.NewDataStore(&storage.Config{ + Type: storage.TypeMemory, + }, testScope.NewSubScope("flytectl")) + assert.Nil(t, err) + Client = s + err = uploadFastRegisterArtifact(ctx, "testdata/flytesnacksre.tgz", "", "") + assert.NotNil(t, err) + }) +} + +func TestGetStorageClient(t *testing.T) { + t.Run("Failed to create storage client", func(t *testing.T) { + Client = nil + s, err := getStorageClient(ctx) + assert.NotNil(t, err) + assert.Nil(t, s) + }) +} + +func TestGetFlyteTestManifest(t *testing.T) { + t.Run("Failed to get manifest with wrong name", func(t *testing.T) { + _, tag, err := getFlyteTestManifest("no////ne", "no////ne") + assert.NotNil(t, err) + assert.Equal(t, len(tag), 0) + }) + t.Run("Failed to get release", func(t *testing.T) { + _, tag, err := getFlyteTestManifest("flyteorg", "homebrew-tap") + assert.NotNil(t, err) + assert.Equal(t, len(tag), 0) + }) + t.Run("Failed to get manifest", func(t *testing.T) { + flyteManifest = "" + _, tag, err := getFlyteTestManifest("flyteorg", "flytesnacks") + assert.NotNil(t, err) + assert.Equal(t, len(tag), 0) + }) +} + +func TestRegister(t *testing.T) { + t.Run("Failed to register", func(t *testing.T) { + setup() + registerFilesSetup() + node := &admin.NodeExecution{} + err := register(ctx, node, cmdCtx) + assert.NotNil(t, err) + }) +} + +func TestHydrateNode(t *testing.T) { + t.Run("Failed hydrate node", func(t *testing.T) { + setup() + registerFilesSetup() + node := &core.Node{} + err := hydrateNode(node) + assert.NotNil(t, err) + }) + + t.Run("hydrateSpec with wrong type", func(t *testing.T) { + setup() + registerFilesSetup() + task := &admin.Task{} + err := hydrateSpec(task, "") + assert.NotNil(t, err) + }) } diff --git a/flytectl/cmd/register/testdata/flyte-package.tgz b/flytectl/cmd/register/testdata/flyte-package.tgz new file mode 100644 index 0000000000..8ae02691be Binary files /dev/null and b/flytectl/cmd/register/testdata/flyte-package.tgz differ diff --git a/flytectl/cmd/register/testdata/flytesnacks-core.tgz b/flytectl/cmd/register/testdata/flytesnacks-core.tgz new file mode 100644 index 0000000000..4c300c4bba Binary files /dev/null and b/flytectl/cmd/register/testdata/flytesnacks-core.tgz differ diff --git a/flytectl/cmd/register/testdata/invalid-fast.tgz b/flytectl/cmd/register/testdata/invalid-fast.tgz new file mode 100644 index 0000000000..28216a03e5 Binary files /dev/null and b/flytectl/cmd/register/testdata/invalid-fast.tgz differ diff --git a/flytectl/cmd/register/testdata/valid-fast-register.tgz b/flytectl/cmd/register/testdata/valid-fast-register.tgz new file mode 100644 index 0000000000..8ce7998189 Binary files /dev/null and b/flytectl/cmd/register/testdata/valid-fast-register.tgz differ diff --git a/flytectl/cmd/sandbox/start.go b/flytectl/cmd/sandbox/start.go index 559c623a08..807cfe52fc 100644 --- a/flytectl/cmd/sandbox/start.go +++ b/flytectl/cmd/sandbox/start.go @@ -65,10 +65,10 @@ func startSandbox(ctx context.Context, cli docker.Docker, reader io.Reader) (*bu return nil, err } - if len(sandboxConfig.DefaultConfig.SourcesPath) > 0 { + if len(sandboxConfig.DefaultConfig.Source) > 0 { docker.Volumes = append(docker.Volumes, mount.Mount{ Type: mount.TypeBind, - Source: sandboxConfig.DefaultConfig.SourcesPath, + Source: sandboxConfig.DefaultConfig.Source, Target: docker.FlyteSnackDir, }) } diff --git a/flytectl/cmd/sandbox/start_test.go b/flytectl/cmd/sandbox/start_test.go index 36e0399b51..a0bef27c8d 100644 --- a/flytectl/cmd/sandbox/start_test.go +++ b/flytectl/cmd/sandbox/start_test.go @@ -61,10 +61,10 @@ func TestStartSandboxFunc(t *testing.T) { errCh := make(chan error) bodyStatus := make(chan container.ContainerWaitOKBody) mockDocker := &mocks.Docker{} - sandboxConfig.DefaultConfig.SourcesPath = f.UserHomeDir() + sandboxConfig.DefaultConfig.Source = f.UserHomeDir() volumes := append(docker.Volumes, mount.Mount{ Type: mount.TypeBind, - Source: sandboxConfig.DefaultConfig.SourcesPath, + Source: sandboxConfig.DefaultConfig.Source, Target: docker.FlyteSnackDir, }) mockDocker.OnContainerCreate(ctx, &container.Config{ @@ -97,10 +97,10 @@ func TestStartSandboxFunc(t *testing.T) { errCh := make(chan error) bodyStatus := make(chan container.ContainerWaitOKBody) mockDocker := &mocks.Docker{} - sandboxConfig.DefaultConfig.SourcesPath = f.UserHomeDir() + sandboxConfig.DefaultConfig.Source = f.UserHomeDir() volumes := append(docker.Volumes, mount.Mount{ Type: mount.TypeBind, - Source: sandboxConfig.DefaultConfig.SourcesPath, + Source: sandboxConfig.DefaultConfig.Source, Target: docker.FlyteSnackDir, }) mockDocker.OnContainerCreate(ctx, &container.Config{ @@ -133,10 +133,10 @@ func TestStartSandboxFunc(t *testing.T) { errCh := make(chan error) bodyStatus := make(chan container.ContainerWaitOKBody) mockDocker := &mocks.Docker{} - sandboxConfig.DefaultConfig.SourcesPath = f.UserHomeDir() + sandboxConfig.DefaultConfig.Source = f.UserHomeDir() volumes := append(docker.Volumes, mount.Mount{ Type: mount.TypeBind, - Source: sandboxConfig.DefaultConfig.SourcesPath, + Source: sandboxConfig.DefaultConfig.Source, Target: docker.FlyteSnackDir, }) mockDocker.OnContainerCreate(ctx, &container.Config{ @@ -177,10 +177,10 @@ func TestStartSandboxFunc(t *testing.T) { errCh := make(chan error) bodyStatus := make(chan container.ContainerWaitOKBody) mockDocker := &mocks.Docker{} - sandboxConfig.DefaultConfig.SourcesPath = f.UserHomeDir() + sandboxConfig.DefaultConfig.Source = f.UserHomeDir() volumes := append(docker.Volumes, mount.Mount{ Type: mount.TypeBind, - Source: sandboxConfig.DefaultConfig.SourcesPath, + Source: sandboxConfig.DefaultConfig.Source, Target: docker.FlyteSnackDir, }) mockDocker.OnContainerCreate(ctx, &container.Config{ @@ -213,10 +213,10 @@ func TestStartSandboxFunc(t *testing.T) { errCh := make(chan error) bodyStatus := make(chan container.ContainerWaitOKBody) mockDocker := &mocks.Docker{} - sandboxConfig.DefaultConfig.SourcesPath = f.UserHomeDir() + sandboxConfig.DefaultConfig.Source = f.UserHomeDir() volumes := append(docker.Volumes, mount.Mount{ Type: mount.TypeBind, - Source: sandboxConfig.DefaultConfig.SourcesPath, + Source: sandboxConfig.DefaultConfig.Source, Target: docker.FlyteSnackDir, }) mockDocker.OnContainerCreate(ctx, &container.Config{ @@ -249,10 +249,10 @@ func TestStartSandboxFunc(t *testing.T) { errCh := make(chan error) bodyStatus := make(chan container.ContainerWaitOKBody) mockDocker := &mocks.Docker{} - sandboxConfig.DefaultConfig.SourcesPath = f.UserHomeDir() + sandboxConfig.DefaultConfig.Source = f.UserHomeDir() volumes := append(docker.Volumes, mount.Mount{ Type: mount.TypeBind, - Source: sandboxConfig.DefaultConfig.SourcesPath, + Source: sandboxConfig.DefaultConfig.Source, Target: docker.FlyteSnackDir, }) mockDocker.OnContainerCreate(ctx, &container.Config{ @@ -312,7 +312,7 @@ func TestStartSandboxFunc(t *testing.T) { }).Return(reader, nil) mockDocker.OnContainerWaitMatch(ctx, mock.Anything, container.WaitConditionNotRunning).Return(bodyStatus, errCh) docker.Client = mockDocker - sandboxConfig.DefaultConfig.SourcesPath = "" + sandboxConfig.DefaultConfig.Source = "" err := startSandboxCluster(ctx, []string{}, cmdCtx) assert.Nil(t, err) }) @@ -348,7 +348,7 @@ func TestStartSandboxFunc(t *testing.T) { }).Return(reader, nil) mockDocker.OnContainerWaitMatch(ctx, mock.Anything, container.WaitConditionNotRunning).Return(bodyStatus, errCh) docker.Client = mockDocker - sandboxConfig.DefaultConfig.SourcesPath = "" + sandboxConfig.DefaultConfig.Source = "" err := startSandboxCluster(ctx, []string{}, cmdCtx) assert.NotNil(t, err) }) diff --git a/flytectl/config.yaml b/flytectl/config.yaml index eccd8b7498..b3710406ef 100644 --- a/flytectl/config.yaml +++ b/flytectl/config.yaml @@ -4,4 +4,15 @@ admin: insecure: true logger: show-source: true - level: 1 + level: 3 +storage: + connection: + access-key: minio + auth-type: accesskey + disable-ssl: true + endpoint: http://localhost:30084 + region: us-east-1 + secret-key: miniostorage + type: minio + container: "my-s3-bucket" + enable-multicontainer: true \ No newline at end of file diff --git a/flytectl/go.mod b/flytectl/go.mod index f944444102..43025c7c76 100644 --- a/flytectl/go.mod +++ b/flytectl/go.mod @@ -9,6 +9,7 @@ require ( github.com/docker/docker v20.10.7+incompatible github.com/docker/go-connections v0.4.0 github.com/enescakir/emoji v1.0.0 + github.com/flyteorg/datacatalog v0.3.6 github.com/flyteorg/flyteidl v0.19.3 github.com/flyteorg/flytestdlib v0.3.24 github.com/ghodss/yaml v1.0.0 @@ -29,6 +30,7 @@ require ( github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect github.com/opencontainers/image-spec v1.0.1 github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4 + github.com/pkg/errors v0.9.1 // indirect github.com/sirupsen/logrus v1.8.0 github.com/spf13/cobra v1.1.3 github.com/spf13/pflag v1.0.5 @@ -36,7 +38,7 @@ require ( github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0 github.com/zalando/go-keyring v0.1.1 golang.org/x/oauth2 v0.0.0-20210126194326-f9ce19ea3013 - google.golang.org/grpc v1.35.0 + google.golang.org/grpc v1.36.0 google.golang.org/protobuf v1.25.0 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect gopkg.in/yaml.v2 v2.4.0 diff --git a/flytectl/go.sum b/flytectl/go.sum index 6658e96614..46b9491b38 100644 --- a/flytectl/go.sum +++ b/flytectl/go.sum @@ -78,6 +78,8 @@ github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBp github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= +github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc= +github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA= github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59jBh3ajtinzw= @@ -102,6 +104,7 @@ github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/Selvatico/go-mocket v1.0.7/go.mod h1:4gO2v+uQmsL+jzQgLANy3tyEFzaEzHlymVbZ3GP2Oes= github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d/go.mod h1:HI8ITrYtUY+O+ZhtlqUnD8+KwNPOyugEhfP9fdUIaEQ= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= @@ -113,6 +116,7 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/alexflint/go-filemutex v0.0.0-20171022225611-72bdc8eae2ae/go.mod h1:CgnQgUtFrFz9mxFNtED3jI5tLDjKlOM+oUF/sTk6ps0= +github.com/antihax/optional v0.0.0-20180407024304-ca021399b1a6/go.mod h1:V8iCPQYkqmusNa815XgQio277wI47sdRh1dUOLdyC6Q= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= @@ -170,6 +174,8 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I= +github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/containerd/aufs v0.0.0-20200908144142-dab0cbea06f4/go.mod h1:nukgQABAEopAHvB6j7cnP5zJ+/3aVcE7hCYqvIwAHyE= @@ -267,6 +273,7 @@ github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee github.com/coreos/go-systemd v0.0.0-20161114122254-48702e0da86b/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd/v22 v22.0.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= github.com/coreos/go-systemd/v22 v22.1.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= @@ -337,6 +344,9 @@ github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv github.com/fatih/color v1.10.0 h1:s36xzo75JdqLaaWoiEHk767eHiwo0598uUxyfiPkDsg= github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= github.com/fatih/structtag v1.2.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4/aAZl94= +github.com/flyteorg/datacatalog v0.3.6 h1:eMA9FK1m7bmOp0LRPt3oT7N+Ki2k6R2IzsqmC69/O8Q= +github.com/flyteorg/datacatalog v0.3.6/go.mod h1:BFc3bTKlvQAGRN0aZDvCt8u1tOX39yN9bcI3R7C4hTY= +github.com/flyteorg/flyteidl v0.18.17/go.mod h1:b5Fq4Z8a5b0mF6pEwTd48ufvikUGVkWSjZiMT0ZtqKI= github.com/flyteorg/flyteidl v0.19.3 h1:Rba35Qi3eG/lExzWUnjQOgX+HpHHLWj+N/QBw9TlFBo= github.com/flyteorg/flyteidl v0.19.3/go.mod h1:576W2ViEyjTpT+kEVHAGbrTP3HARNUZ/eCwrNPmdx9U= github.com/flyteorg/flytestdlib v0.3.13/go.mod h1:Tz8JCECAbX6VWGwFT6cmEQ+RJpZ/6L9pswu3fzWs220= @@ -386,6 +396,8 @@ github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e h1:BWhy2j3IXJhjCbC68Fp github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4= github.com/godbus/dbus/v5 v5.0.3 h1:ZqHaoEF7TBzh4jzPmqVhE/5A1z9of6orkAe5uHoAeME= github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gofrs/uuid v3.2.0+incompatible h1:y12jRkkFxsd7GpqdSZ+/KCs/fJbqpEXSGd4+jfEaewE= +github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= github.com/gogo/googleapis v1.2.0/go.mod h1:Njal3psf3qN6dwBtQfUmBZh2ybovJ0tlu3o/AC7HYjU= github.com/gogo/googleapis v1.4.0/go.mod h1:5YRNX2z1oM5gXdAkurHa942MDgEJyk02w4OecKY87+c= @@ -497,6 +509,7 @@ github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92Bcuy github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/grpc-ecosystem/grpc-gateway v1.12.2/go.mod h1:8XEsbTttt/W+VvjtQhLACqCisSPWTxCZ7sBRjU6iH9c= github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= @@ -540,6 +553,62 @@ github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NH github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56/go.mod h1:ymszkNOg6tORTn+6F6j+Jc8TOr5osrynvN6ivFWZ2GA= +github.com/jackc/chunkreader v1.0.0 h1:4s39bBR8ByfqH+DKm8rQA3E1LHZWB9XWcrz8fqaZbe0= +github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo= +github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8= +github.com/jackc/chunkreader/v2 v2.0.1/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/pgconn v0.0.0-20190420214824-7e0022ef6ba3/go.mod h1:jkELnwuX+w9qN5YIfX0fl88Ehu4XC3keFuOJJk9pcnA= +github.com/jackc/pgconn v0.0.0-20190824142844-760dd75542eb/go.mod h1:lLjNuW/+OfW9/pnVKPazfWOgNfH2aPem8YQ7ilXGvJE= +github.com/jackc/pgconn v0.0.0-20190831204454-2fabfa3c18b7/go.mod h1:ZJKsE/KZfsUgOEh9hBm+xYTstcNHg7UPMVJqRfQxq4s= +github.com/jackc/pgconn v1.4.0/go.mod h1:Y2O3ZDF0q4mMacyWV3AstPJpeHXWGEetiFttmq5lahk= +github.com/jackc/pgconn v1.5.0/go.mod h1:QeD3lBfpTFe8WUnPZWN5KY/mB8FGMIYRdd8P8Jr0fAI= +github.com/jackc/pgconn v1.5.1-0.20200601181101-fa742c524853/go.mod h1:QeD3lBfpTFe8WUnPZWN5KY/mB8FGMIYRdd8P8Jr0fAI= +github.com/jackc/pgconn v1.8.1 h1:ySBX7Q87vOMqKU2bbmKbUvtYhauDFclYbNDYIE1/h6s= +github.com/jackc/pgconn v1.8.1/go.mod h1:JV6m6b6jhjdmzchES0drzCcYcAHS1OPD5xu3OZ/lE2g= +github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= +github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= +github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2 h1:JVX6jT/XfzNqIjye4717ITLaNwV9mWbJx0dLCpcRzdA= +github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgproto3 v1.1.0 h1:FYYE4yRw+AgI8wXIinMlNjBbp/UitDJwfj5LqqewP1A= +github.com/jackc/pgproto3 v1.1.0/go.mod h1:eR5FA3leWg7p9aeAqi37XOTgTIbkABlvcPB3E5rlc78= +github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190420180111-c116219b62db/go.mod h1:bhq50y+xrl9n5mRYyCBFKkpRVTLYJVWeCc+mEAI3yXA= +github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190609003834-432c2951c711/go.mod h1:uH0AWtUmuShn0bcesswc4aBTWGvw0cAxIJp+6OB//Wg= +github.com/jackc/pgproto3/v2 v2.0.0-rc3/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= +github.com/jackc/pgproto3/v2 v2.0.0-rc3.0.20190831210041-4c03ce451f29/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= +github.com/jackc/pgproto3/v2 v2.0.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgproto3/v2 v2.0.6 h1:b1105ZGEMFe7aCvrT1Cca3VoVb4ZFMaFJLJcg/3zD+8= +github.com/jackc/pgproto3/v2 v2.0.6/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgservicefile v0.0.0-20200307190119-3430c5407db8/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= +github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b h1:C8S2+VttkHFdOOCXJe+YGfa4vHYwlt4Zx+IVXQ97jYg= +github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= +github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01CGwFsrv11mJRHWJ6aifDLfdV3aVjFF0zg= +github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc= +github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw= +github.com/jackc/pgtype v1.2.0/go.mod h1:5m2OfMh1wTK7x+Fk952IDmI4nw3nPrvtQdM0ZT4WpC0= +github.com/jackc/pgtype v1.3.1-0.20200510190516-8cd94a14c75a/go.mod h1:vaogEUkALtxZMCH411K+tKzNpwzCKU+AnPzBKZ+I+Po= +github.com/jackc/pgtype v1.3.1-0.20200606141011-f6355165a91c/go.mod h1:cvk9Bgu/VzJ9/lxTO5R5sf80p0DiucVtN7ZxvaC4GmQ= +github.com/jackc/pgtype v1.7.0 h1:6f4kVsW01QftE38ufBYxKciO6gyioXSC0ABIRLcZrGs= +github.com/jackc/pgtype v1.7.0/go.mod h1:ZnHF+rMePVqDKaOfJVI4Q8IVvAQMryDlDkZnKOI75BE= +github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y= +github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM= +github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc= +github.com/jackc/pgx/v4 v4.5.0/go.mod h1:EpAKPLdnTorwmPUUsqrPxy5fphV18j9q3wrfRXgo+kA= +github.com/jackc/pgx/v4 v4.6.1-0.20200510190926-94ba730bb1e9/go.mod h1:t3/cdRQl6fOLDxqtlyhe9UWgfIi9R8+8v8GKV5TRA/o= +github.com/jackc/pgx/v4 v4.6.1-0.20200606145419-4e5062306904/go.mod h1:ZDaNWkt9sW1JMiNn0kdYBaLelIhw7Pg4qd+Vk6tw7Hg= +github.com/jackc/pgx/v4 v4.11.0 h1:J86tSWd3Y7nKjwT/43xZBvpi04keQWx8gNC2YkdJhZI= +github.com/jackc/pgx/v4 v4.11.0/go.mod h1:i62xJgdrtVDsnL3U8ekyrQXEwGNTRoG7/8r+CIdYfcc= +github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.1.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.1.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= +github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= +github.com/jinzhu/now v1.1.2 h1:eVKgfIdy9b6zbWBMgFpfDPoAMifwSZagU9HmEU6zgiI= +github.com/jinzhu/now v1.1.2/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.0.0-20160803190731-bd40a432e4c7/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= @@ -578,11 +647,17 @@ github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfn github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= +github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/landoop/tableprinter v0.0.0-20180806200924-8bd8c2576d27 h1:O664tckOIC4smyHDDJPXAh/YBYYc0Y1O8S5wmZDm3d8= github.com/landoop/tableprinter v0.0.0-20180806200924-8bd8c2576d27/go.mod h1:f0X1c0za3TbET/rl5ThtCSel0+G3/yZ8iuU9BxnyVK0= +github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.3.0 h1:/qkRGz8zljWiDcFvgpwUpwIAPu3r07TDvs3Rws+o/pU= +github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= @@ -597,10 +672,17 @@ github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= github.com/marstr/guid v1.1.0/go.mod h1:74gB1z2wpxxInTG6yaqA7KrtM0NZ+RbrcqDvYHefzho= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= +github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8= github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= @@ -732,6 +814,7 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= +github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= github.com/pquerna/ffjson v0.0.0-20190813045741-dac163c6c0a9/go.mod h1:YARuvh7BUWHNhzDq2OM5tzR2RiCcN2D7sapiKyCel/M= github.com/prometheus/client_golang v0.0.0-20180209125602-c332b6f63c06/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= @@ -780,6 +863,9 @@ github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqn github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= +github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= +github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= @@ -789,6 +875,9 @@ github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo= +github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= +github.com/shopspring/decimal v0.0.0-20200227202807-02e2044944cc h1:jUIKcSPO9MoMJBbEoyE/RJoE8vz7Mb8AjvifMMwSyvY= +github.com/shopspring/decimal v0.0.0-20200227202807-02e2044944cc/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.0.4-0.20170822132746-89742aefa4b2/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= @@ -890,6 +979,7 @@ github.com/yvasiyarov/gorelic v0.0.0-20141212073537-a9bba5b9ab50/go.mod h1:NUSPS github.com/yvasiyarov/newrelic_platform_go v0.0.0-20140908184405-b21fdbd4370f/go.mod h1:GlGEuHIJweS1mbCqG+7vt2nvWLzLLnRHbXz5JKd/Qbg= github.com/zalando/go-keyring v0.1.1 h1:w2V9lcx/Uj4l+dzAf1m9s+DJ1O8ROkEHnynonHjTcYE= github.com/zalando/go-keyring v0.1.1/go.mod h1:OIC+OZ28XbmwFxU/Rp9V7eKzZjamBJwRzC8UFJH9+L8= +github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= @@ -909,9 +999,12 @@ go.opencensus.io v0.22.6/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= +go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= +go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= golang.org/x/crypto v0.0.0-20171113213409-9f005a07e0d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -919,12 +1012,15 @@ golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20181009213950-7c1a557ab941/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -990,6 +1086,7 @@ golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191002035440-2ec189313ef0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -1044,7 +1141,9 @@ golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1059,6 +1158,7 @@ golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190812073006-9eafafc0a87e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1141,6 +1241,7 @@ golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= @@ -1149,6 +1250,7 @@ golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190823170909-c4a336ef6a2f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -1193,6 +1295,8 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0 h1:po9/4sTYwZU9lPhi1tOrb4hCv3qrhiQ77LZfGa2OjwY= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1242,6 +1346,7 @@ google.golang.org/genproto v0.0.0-20190530194941-fb225487d101/go.mod h1:z3L6/3dT google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20190927181202-20e1ac93f88c/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= @@ -1301,8 +1406,9 @@ google.golang.org/grpc v1.32.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= -google.golang.org/grpc v1.35.0 h1:TwIQcH3es+MojMVojxxfQ3l3OF2KzlRxML2xZq0kRo8= google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.0 h1:o1bcQ6imQMIOpdrO3SWf2z5RV72WbDwdXuK0MDlc8As= +google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -1327,6 +1433,7 @@ gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= +gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.62.0 h1:duBzk771uxoUuOlyRLkHsygud9+5lrlGjdFBb4mSKDU= @@ -1336,6 +1443,7 @@ gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24 gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/square/go-jose.v2 v2.3.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= +gopkg.in/square/go-jose.v2 v2.4.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/square/go-jose.v2 v2.5.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= @@ -1352,6 +1460,10 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gorm.io/driver/postgres v1.1.0 h1:afBljg7PtJ5lA6YUWluV2+xovIPhS+YiInuL3kUjrbk= +gorm.io/driver/postgres v1.1.0/go.mod h1:hXQIwafeRjJvUm+OMxcFWyswJ/vevcpPLlGocwAwuqw= +gorm.io/gorm v1.21.9 h1:INieZtn4P2Pw6xPJ8MzT0G4WUOsHq3RhfuDF1M6GW0E= +gorm.io/gorm v1.21.9/go.mod h1:F+OptMscr0P2F2qU97WT1WimdH9GaQPoDW7AYd5i2Y0= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=