diff --git a/flytestdlib/cli/pflags/api/generator_test.go b/flytestdlib/cli/pflags/api/generator_test.go index 21eee3b026..0990382245 100644 --- a/flytestdlib/cli/pflags/api/generator_test.go +++ b/flytestdlib/cli/pflags/api/generator_test.go @@ -3,7 +3,6 @@ package api import ( "context" "flag" - "fmt" "go/token" "go/types" "io/ioutil" @@ -174,8 +173,8 @@ func TestBuildFieldForMap(t *testing.T) { defaultValue := "" fieldInfo, err := buildFieldForMap(ctx, typesMap, name, goName, usage, defaultValue, false) assert.NotNil(t, err) - assert.Equal(t, fmt.Errorf("map of type [interface{/* incomplete */}] is not supported."+ - " Only basic slices or slices of json-unmarshalable types are supported"), err) + assert.Contains(t, err.Error(), "is not supported."+ + " Only basic slices or slices of json-unmarshalable types are supported") assert.NotNil(t, fieldInfo) assert.Equal(t, "", fieldInfo.FlagMethodName) assert.Equal(t, "", fieldInfo.DefaultValue) diff --git a/flytestdlib/cli/pflags/api/testdata/testtype.go b/flytestdlib/cli/pflags/api/testdata/testtype.go index 4a86fc15ef..5f95e2ce95 100755 --- a/flytestdlib/cli/pflags/api/testdata/testtype.go +++ b/flytestdlib/cli/pflags/api/testdata/testtype.go @@ -64,8 +64,8 @@ func (cfg TestType) GetPFlagSet(prefix string) *pflag.FlagSet { cmdFlags.String(fmt.Sprintf("%v%v", prefix, "storage.connection.secret-key"), DefaultTestType.StorageConfig.Connection.SecretKey, "Secret to use when accesskey is set.") cmdFlags.String(fmt.Sprintf("%v%v", prefix, "storage.connection.region"), DefaultTestType.StorageConfig.Connection.Region, "Region to connect to.") cmdFlags.Bool(fmt.Sprintf("%v%v", prefix, "storage.connection.disable-ssl"), DefaultTestType.StorageConfig.Connection.DisableSSL, "Disables SSL connection. Should only be used for development.") - cmdFlags.String(fmt.Sprintf("%v%v", prefix, "storage.stow.kind"), DefaultTestType.StorageConfig.Stow.Kind, "Kind of Stow backend to use. Refer to github/graymeta/stow") - cmdFlags.StringToString(fmt.Sprintf("%v%v", prefix, "storage.stow.config"), DefaultTestType.StorageConfig.Stow.Config, "Configuration for stow backend. Refer to github/graymeta/stow") + cmdFlags.String(fmt.Sprintf("%v%v", prefix, "storage.stow.kind"), DefaultTestType.StorageConfig.Stow.Kind, "Kind of Stow backend to use. Refer to github/flyteorg/stow") + cmdFlags.StringToString(fmt.Sprintf("%v%v", prefix, "storage.stow.config"), DefaultTestType.StorageConfig.Stow.Config, "Configuration for stow backend. Refer to github/flyteorg/stow") cmdFlags.String(fmt.Sprintf("%v%v", prefix, "storage.container"), DefaultTestType.StorageConfig.InitContainer, "Initial container (in s3 a bucket) to create -if it doesn't exist-.'") cmdFlags.Bool(fmt.Sprintf("%v%v", prefix, "storage.enable-multicontainer"), DefaultTestType.StorageConfig.MultiContainerEnabled, "If this is true, then the container argument is overlooked and redundant. This config will automatically open new connections to new containers/buckets as they are encountered") cmdFlags.Int(fmt.Sprintf("%v%v", prefix, "storage.cache.max_size_mbs"), DefaultTestType.StorageConfig.Cache.MaxSizeMegabytes, "Maximum size of the cache where the Blob store data is cached in-memory. If not specified or set to 0, cache is not used") diff --git a/flytestdlib/cli/pflags/api/testdata/testtype_bind.go b/flytestdlib/cli/pflags/api/testdata/testtype_bind.go index e12dc052ee..4fef798cce 100755 --- a/flytestdlib/cli/pflags/api/testdata/testtype_bind.go +++ b/flytestdlib/cli/pflags/api/testdata/testtype_bind.go @@ -64,8 +64,8 @@ func (cfg TestType) GetPFlagSet(prefix string) *pflag.FlagSet { cmdFlags.StringVar(&DefaultTestType.StorageConfig.Connection.SecretKey, fmt.Sprintf("%v%v", prefix, "storage.connection.secret-key"), DefaultTestType.StorageConfig.Connection.SecretKey, "Secret to use when accesskey is set.") cmdFlags.StringVar(&DefaultTestType.StorageConfig.Connection.Region, fmt.Sprintf("%v%v", prefix, "storage.connection.region"), DefaultTestType.StorageConfig.Connection.Region, "Region to connect to.") cmdFlags.BoolVar(&DefaultTestType.StorageConfig.Connection.DisableSSL, fmt.Sprintf("%v%v", prefix, "storage.connection.disable-ssl"), DefaultTestType.StorageConfig.Connection.DisableSSL, "Disables SSL connection. Should only be used for development.") - cmdFlags.StringVar(&DefaultTestType.StorageConfig.Stow.Kind, fmt.Sprintf("%v%v", prefix, "storage.stow.kind"), DefaultTestType.StorageConfig.Stow.Kind, "Kind of Stow backend to use. Refer to github/graymeta/stow") - cmdFlags.StringToStringVar(&DefaultTestType.StorageConfig.Stow.Config, fmt.Sprintf("%v%v", prefix, "storage.stow.config"), DefaultTestType.StorageConfig.Stow.Config, "Configuration for stow backend. Refer to github/graymeta/stow") + cmdFlags.StringVar(&DefaultTestType.StorageConfig.Stow.Kind, fmt.Sprintf("%v%v", prefix, "storage.stow.kind"), DefaultTestType.StorageConfig.Stow.Kind, "Kind of Stow backend to use. Refer to github/flyteorg/stow") + cmdFlags.StringToStringVar(&DefaultTestType.StorageConfig.Stow.Config, fmt.Sprintf("%v%v", prefix, "storage.stow.config"), DefaultTestType.StorageConfig.Stow.Config, "Configuration for stow backend. Refer to github/flyteorg/stow") cmdFlags.StringVar(&DefaultTestType.StorageConfig.InitContainer, fmt.Sprintf("%v%v", prefix, "storage.container"), DefaultTestType.StorageConfig.InitContainer, "Initial container (in s3 a bucket) to create -if it doesn't exist-.'") cmdFlags.BoolVar(&DefaultTestType.StorageConfig.MultiContainerEnabled, fmt.Sprintf("%v%v", prefix, "storage.enable-multicontainer"), DefaultTestType.StorageConfig.MultiContainerEnabled, "If this is true, then the container argument is overlooked and redundant. This config will automatically open new connections to new containers/buckets as they are encountered") cmdFlags.IntVar(&DefaultTestType.StorageConfig.Cache.MaxSizeMegabytes, fmt.Sprintf("%v%v", prefix, "storage.cache.max_size_mbs"), DefaultTestType.StorageConfig.Cache.MaxSizeMegabytes, "Maximum size of the cache where the Blob store data is cached in-memory. If not specified or set to 0, cache is not used") diff --git a/flytestdlib/go.mod b/flytestdlib/go.mod index 89d9934229..828f3b5b2f 100644 --- a/flytestdlib/go.mod +++ b/flytestdlib/go.mod @@ -25,7 +25,7 @@ require ( github.com/spf13/viper v1.7.1 github.com/stretchr/testify v1.7.0 golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 - golang.org/x/tools v0.1.0 + golang.org/x/tools v0.1.10 google.golang.org/protobuf v1.25.0 gorm.io/gorm v1.22.4 k8s.io/api v0.20.2 @@ -79,12 +79,12 @@ require ( github.com/stretchr/objx v0.3.0 // indirect github.com/subosito/gotenv v1.2.0 // indirect go.opencensus.io v0.22.6 // indirect - golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad // indirect + golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 // indirect golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5 // indirect - golang.org/x/mod v0.4.1 // indirect - golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d // indirect + golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 // indirect + golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f // indirect golang.org/x/oauth2 v0.0.0-20210126194326-f9ce19ea3013 // indirect - golang.org/x/sys v0.0.0-20210423082822-04245dca01da // indirect + golang.org/x/sys v0.0.0-20211019181941-9d821ace8654 // indirect golang.org/x/text v0.3.7 // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect google.golang.org/api v0.38.0 // indirect diff --git a/flytestdlib/go.sum b/flytestdlib/go.sum index 155bd611bc..6e2d1974e0 100644 --- a/flytestdlib/go.sum +++ b/flytestdlib/go.sum @@ -49,19 +49,27 @@ github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.3.0 h1:Px2UA+2RvSSvv+RvJ github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.3.0/go.mod h1:tPaiy8S5bQ+S5sOiDlINkp7+Ef339+Nz5L5XO+cnOHo= github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= +github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= github.com/Azure/go-autorest/autorest v0.11.12/go.mod h1:eipySxLmqSyC5s5k1CLupqet0PSENBEDP93LQ9a8QYw= github.com/Azure/go-autorest/autorest v0.11.17 h1:2zCdHwNgRH+St1J+ZMf66xI8aLr/5KMy+wWLH97zwYM= github.com/Azure/go-autorest/autorest v0.11.17/go.mod h1:eipySxLmqSyC5s5k1CLupqet0PSENBEDP93LQ9a8QYw= +github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0= github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A= github.com/Azure/go-autorest/autorest/adal v0.9.10 h1:r6fZHMaHD8B6LDCn0o5vyBFHIHrM6Ywwx7mb49lPItI= github.com/Azure/go-autorest/autorest/adal v0.9.10/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A= +github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA= github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw= github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= +github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= +github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= github.com/Azure/go-autorest/autorest/mocks v0.4.1 h1:K0laFcLE6VLTOwNgSxaGbUcLPuGXlNkbVvq4cW4nIHk= github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= github.com/Azure/go-autorest/autorest/to v0.4.0 h1:oXVqrxakqqV1UZdSazDOPOLvOIz+XA683u8EctwboHk= +github.com/Azure/go-autorest/autorest/to v0.4.0/go.mod h1:fE8iZBn7LQR7zH/9XU2NcPR4o9jEImooCeWJcYV/zLE= +github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc= github.com/Azure/go-autorest/logger v0.2.0 h1:e4RVHVZKC5p6UANLJHkM4OfR1UKZPj8Wt8Pcx+3oqrE= github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= +github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= @@ -89,6 +97,7 @@ github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= +github.com/aws/aws-sdk-go v1.23.4/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.37.1 h1:BTHmuN+gzhxkvU9sac2tZvaY0gV9ihbHw+KxZOecYvY= github.com/aws/aws-sdk-go v1.37.1/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= @@ -109,6 +118,7 @@ github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghf github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927 h1:SKI1/fuSdodxmNNyVBR8d7X/HuLnRpvvFO0AgyQk764= +github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927/go.mod h1:h/aW8ynjgkuj+NQRlZcDbAbM1ORAbXjXX77sX7T289U= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= @@ -271,6 +281,7 @@ github.com/google/pprof v0.0.0-20200905233945-acf8798be1f7/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/readahead v0.0.0-20161222183148-eaceba169032/go.mod h1:qYysrqQXuV4tzsizt4oOQ6mrBZQ0xnQXP3ylXX8Jk5Y= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -412,6 +423,7 @@ github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzE github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +github.com/ncw/swift v1.0.49/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM= github.com/ncw/swift v1.0.53 h1:luHjjTNtekIEvHg5KdAFIBaH7bWfNkefwFnpDffSIks= github.com/ncw/swift v1.0.53/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM= github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs= @@ -449,10 +461,12 @@ github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= +github.com/pkg/sftp v1.10.0/go.mod h1:NxmoDg/QLVWluQDUYG7XBZTLUpKeFa8e3aMf1BfjyHk= github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= +github.com/pquerna/ffjson v0.0.0-20190813045741-dac163c6c0a9/go.mod h1:YARuvh7BUWHNhzDq2OM5tzR2RiCcN2D7sapiKyCel/M= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= @@ -554,6 +568,7 @@ github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= @@ -586,8 +601,8 @@ golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad h1:DN0cp81fZ3njFcrLCytUHRSUkqBjfTo4Tx9RJTWs0EY= -golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 h1:7I4JAnoQBe7ZtJcBaYHi5UtiO8tQHbUSXxL+pnGRANg= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -621,8 +636,8 @@ golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.1 h1:Kvvh58BN8Y9/lBi7hTekvtMpm07eUZ0ck5pRHpsMWrY= -golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 h1:kQgndtyPBW/JIYERgdxfwMYh3AVStj88WQTlNDi2a+o= +golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -665,9 +680,11 @@ golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d h1:20cMwl2fHAzkJMEA+8J4JgqBQcQGzbisXo31MIeenXI= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f h1:OfiFi4JbukWwe3lzw+xunroH1mnC1e2Gy5cxNJApiSY= +golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -688,6 +705,7 @@ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -738,10 +756,10 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201214210602-f9fddec55a1e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da h1:b3NXsE2LusjYGGjL5bxEVZZORm/YEFFrWFjR8eFrw/c= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211019181941-9d821ace8654 h1:id054HUawV2/6IGm2IV8KZQjqtwAOo2CYlOToYqa0d0= +golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -819,8 +837,8 @@ golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.0 h1:po9/4sTYwZU9lPhi1tOrb4hCv3qrhiQ77LZfGa2OjwY= -golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/tools v0.1.10 h1:QjFRCZxdOhBJ/UNgnBZLbNV13DlbnK0quyivTnXJM20= +golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -948,6 +966,7 @@ gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.62.0 h1:duBzk771uxoUuOlyRLkHsygud9+5lrlGjdFBb4mSKDU= gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/kothar/go-backblaze.v0 v0.0.0-20190520213052-702d4e7eb465/go.mod h1:zJ2QpyDCYo1KvLXlmdnFlQAyF/Qfth0fB8239Qg7BIE= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= diff --git a/flytestdlib/storage/config.go b/flytestdlib/storage/config.go index 92aa814ddc..d8c4dd95ff 100644 --- a/flytestdlib/storage/config.go +++ b/flytestdlib/storage/config.go @@ -45,7 +45,8 @@ var ( // Config is a common storage config. type Config struct { - Type Type `json:"type" pflag:",Sets the type of storage to configure [s3/minio/local/mem/stow]."` + Type Type `json:"type" pflag:",Sets the type of storage to configure [s3/minio/local/mem/stow]."` + // Deprecated: Please use StowConfig instead Connection ConnectionConfig `json:"connection"` Stow StowConfig `json:"stow,omitempty" pflag:",Storage config for stow backend."` // Container here is misleading, it refers to a Bucket (AWS S3) like blobstore entity. In some terms it could be a table @@ -60,6 +61,12 @@ type Config struct { Cache CachingConfig `json:"cache"` Limits LimitsConfig `json:"limits" pflag:",Sets limits for stores."` DefaultHTTPClient HTTPClientConfig `json:"defaultHttpClient" pflag:",Sets the default http client config."` + SignedURL SignedURLConfig `json:"signedUrl" pflag:",Sets config for SignedURL."` +} + +// SignedURLConfig encapsulates configs specifically used for SignedURL behavior. +type SignedURLConfig struct { + StowConfigOverride map[string]string `json:"stowConfigOverride,omitempty" pflag:"-,Configuration for stow backend. Refer to github/flyteorg/stow"` } // HTTPClientConfig encapsulates common settings that can be applied to an HTTP Client. @@ -78,9 +85,10 @@ type ConnectionConfig struct { DisableSSL bool `json:"disable-ssl" pflag:",Disables SSL connection. Should only be used for development."` } +// StowConfig defines configs for stow as defined in github.com/flyteorg/stow type StowConfig struct { - Kind string `json:"kind,omitempty" pflag:",Kind of Stow backend to use. Refer to github/graymeta/stow"` - Config map[string]string `json:"config,omitempty" pflag:",Configuration for stow backend. Refer to github/graymeta/stow"` + Kind string `json:"kind,omitempty" pflag:",Kind of Stow backend to use. Refer to github/flyteorg/stow"` + Config map[string]string `json:"config,omitempty" pflag:",Configuration for stow backend. Refer to github/flyteorg/stow"` } type CachingConfig struct { @@ -98,12 +106,12 @@ type CachingConfig struct { TargetGCPercent int `json:"target_gc_percent" pflag:",Sets the garbage collection target percentage."` } -// Specifies limits for storage package. +// LimitsConfig specifies limits for storage package. type LimitsConfig struct { GetLimitMegabytes int64 `json:"maxDownloadMBs" pflag:",Maximum allowed download size (in MBs) per call."` } -// Retrieve current global config for storage. +// GetConfig retrieve current global config for storage. func GetConfig() *Config { if c, ok := ConfigSection.GetConfig().(*Config); ok { return c diff --git a/flytestdlib/storage/config_flags.go b/flytestdlib/storage/config_flags.go index c9c2eb9a10..63a41550c2 100755 --- a/flytestdlib/storage/config_flags.go +++ b/flytestdlib/storage/config_flags.go @@ -57,8 +57,8 @@ func (cfg Config) GetPFlagSet(prefix string) *pflag.FlagSet { cmdFlags.String(fmt.Sprintf("%v%v", prefix, "connection.secret-key"), defaultConfig.Connection.SecretKey, "Secret to use when accesskey is set.") cmdFlags.String(fmt.Sprintf("%v%v", prefix, "connection.region"), defaultConfig.Connection.Region, "Region to connect to.") cmdFlags.Bool(fmt.Sprintf("%v%v", prefix, "connection.disable-ssl"), defaultConfig.Connection.DisableSSL, "Disables SSL connection. Should only be used for development.") - cmdFlags.String(fmt.Sprintf("%v%v", prefix, "stow.kind"), defaultConfig.Stow.Kind, "Kind of Stow backend to use. Refer to github/graymeta/stow") - cmdFlags.StringToString(fmt.Sprintf("%v%v", prefix, "stow.config"), defaultConfig.Stow.Config, "Configuration for stow backend. Refer to github/graymeta/stow") + cmdFlags.String(fmt.Sprintf("%v%v", prefix, "stow.kind"), defaultConfig.Stow.Kind, "Kind of Stow backend to use. Refer to github/flyteorg/stow") + cmdFlags.StringToString(fmt.Sprintf("%v%v", prefix, "stow.config"), defaultConfig.Stow.Config, "Configuration for stow backend. Refer to github/flyteorg/stow") cmdFlags.String(fmt.Sprintf("%v%v", prefix, "container"), defaultConfig.InitContainer, "Initial container (in s3 a bucket) to create -if it doesn't exist-.'") cmdFlags.Bool(fmt.Sprintf("%v%v", prefix, "enable-multicontainer"), defaultConfig.MultiContainerEnabled, "If this is true, then the container argument is overlooked and redundant. This config will automatically open new connections to new containers/buckets as they are encountered") cmdFlags.Int(fmt.Sprintf("%v%v", prefix, "cache.max_size_mbs"), defaultConfig.Cache.MaxSizeMegabytes, "Maximum size of the cache where the Blob store data is cached in-memory. If not specified or set to 0, cache is not used") diff --git a/flytestdlib/storage/stow_store.go b/flytestdlib/storage/stow_store.go index affd9776ce..175389f6f1 100644 --- a/flytestdlib/storage/stow_store.go +++ b/flytestdlib/storage/stow_store.go @@ -5,6 +5,7 @@ import ( "fmt" "io" "net/url" + "strconv" "sync" "time" @@ -108,7 +109,8 @@ func (s StowMetadata) Exists() bool { // Implements DataStore to talk to stow location store. type StowStore struct { copyImpl - loc stow.Location + loc stow.Location + signedURLLoc stow.Location // This is a default configured container. baseContainer stow.Container // If dynamic container loading is enabled, then for any new container that is not the base container @@ -121,8 +123,12 @@ type StowStore struct { } func (s *StowStore) CreateContainer(ctx context.Context, container string) (stow.Container, error) { + return s.createContainer(ctx, locationIDMain, container) +} + +func (s *StowStore) createContainer(ctx context.Context, locID locationID, container string) (stow.Container, error) { logger.Infof(ctx, "Attempting to create container [%s]", container) - c, err := s.loc.CreateContainer(container) + c, err := s.getLocation(locID).CreateContainer(container) if err != nil && !awsBucketAlreadyExists(err) && !IsExists(err) { return nil, fmt.Errorf("unable to initialize container [%v]. Error: %v", container, err) } @@ -130,12 +136,16 @@ func (s *StowStore) CreateContainer(ctx context.Context, container string) (stow } func (s *StowStore) LoadContainer(ctx context.Context, container string, createIfNotFound bool) (stow.Container, error) { - c, err := s.loc.Container(container) + return s.loadContainer(ctx, locationIDMain, container, createIfNotFound) +} + +func (s *StowStore) loadContainer(ctx context.Context, locID locationID, container string, createIfNotFound bool) (stow.Container, error) { + c, err := s.getLocation(locID).Container(container) if err != nil { // IsNotFound is not always guaranteed to be returned if the underlying container doesn't exist! // As of stow v0.2.6, the call to get container elides the lookup when a bucket region is set for S3 containers. if IsNotFound(err) && createIfNotFound { - c, err = s.CreateContainer(ctx, container) + c, err = s.createContainer(ctx, locID, container) if err != nil { logger.Errorf(ctx, "Call to create container [%s] failed. Error %s", container, err) return nil, err @@ -145,29 +155,33 @@ func (s *StowStore) LoadContainer(ctx context.Context, container string, createI return nil, err } } + return c, nil } -func (s *StowStore) getContainer(ctx context.Context, container string) (c stow.Container, err error) { - if s.baseContainer != nil && s.baseContainer.Name() == container { +func (s *StowStore) getContainer(ctx context.Context, locID locationID, container string) (c stow.Container, err error) { + if s.baseContainer != nil && s.baseContainer.Name() == container && locID == locationIDMain { return s.baseContainer, nil } - if !s.enableDynamicContainerLoading { + if !s.enableDynamicContainerLoading && locID == locationIDMain { s.metrics.BadContainer.Inc(ctx) return nil, errs.Wrapf(stow.ErrNotFound, "Conf container:%v != Passed Container:%v. Dynamic loading is disabled", s.baseContainer.Name(), container) } - iface, ok := s.dynamicContainerMap.Load(container) + containerID := locID.String() + container + iface, ok := s.dynamicContainerMap.Load(containerID) if !ok { - c, err := s.LoadContainer(ctx, container, false) + c, err := s.loadContainer(ctx, locID, container, false) if err != nil { logger.Errorf(ctx, "failed to load container [%s] dynamically, error %s", container, err) return nil, err } - s.dynamicContainerMap.Store(container, c) + + s.dynamicContainerMap.Store(containerID, c) return c, nil } + return iface.(stow.Container), nil } @@ -178,7 +192,7 @@ func (s *StowStore) Head(ctx context.Context, reference DataReference) (Metadata return nil, err } - container, err := s.getContainer(ctx, c) + container, err := s.getContainer(ctx, locationIDMain, c) if err != nil { return nil, err } @@ -213,7 +227,7 @@ func (s *StowStore) ReadRaw(ctx context.Context, reference DataReference) (io.Re return nil, err } - container, err := s.getContainer(ctx, c) + container, err := s.getContainer(ctx, locationIDMain, c) if err != nil { return nil, err } @@ -247,7 +261,7 @@ func (s *StowStore) WriteRaw(ctx context.Context, reference DataReference, size return err } - container, err := s.getContainer(ctx, c) + container, err := s.getContainer(ctx, locationIDMain, c) if err != nil { return err } @@ -283,7 +297,7 @@ func (s *StowStore) CreateSignedURL(ctx context.Context, reference DataReference return SignedURLResponse{}, err } - c, err := s.getContainer(ctx, container) + c, err := s.getContainer(ctx, locationIDSignedURL, container) if err != nil { return SignedURLResponse{}, err } @@ -306,10 +320,35 @@ func (s *StowStore) CreateSignedURL(ctx context.Context, reference DataReference }, nil } -func NewStowRawStore(baseContainerFQN DataReference, loc stow.Location, enableDynamicContainerLoading bool, metricsScope promutils.Scope) (*StowStore, error) { +type locationID uint + +const ( + locationIDMain locationID = iota + locationIDSignedURL +) + +func (l locationID) String() string { + return strconv.Itoa(int(l)) +} + +func (s *StowStore) getLocation(id locationID) stow.Location { + switch id { + case locationIDSignedURL: + if s.signedURLLoc != nil { + return s.signedURLLoc + } + + fallthrough + default: + return s.loc + } +} + +func NewStowRawStore(baseContainerFQN DataReference, loc, signedURLLoc stow.Location, enableDynamicContainerLoading bool, metricsScope promutils.Scope) (*StowStore, error) { failureTypeOption := labeled.AdditionalLabelsOption{Labels: []string{FailureTypeLabel.String()}} self := &StowStore{ loc: loc, + signedURLLoc: signedURLLoc, baseContainerFQN: baseContainerFQN, enableDynamicContainerLoading: enableDynamicContainerLoading, dynamicContainerMap: sync.Map{}, @@ -369,7 +408,17 @@ func newStowRawStore(cfg *Config, metricsScope promutils.Scope) (RawStore, error return emptyStore, fmt.Errorf("unable to configure the storage for %s. Error: %v", kind, err) } - return NewStowRawStore(fn(cfg.InitContainer), loc, cfg.MultiContainerEnabled, metricsScope) + var signedURLLoc stow.Location + if len(cfg.SignedURL.StowConfigOverride) > 0 { + var newCfg stow.ConfigMap = make(map[string]string, len(cfgMap)) + MergeMaps(newCfg, cfgMap, cfg.SignedURL.StowConfigOverride) + signedURLLoc, err = stow.Dial(kind, newCfg) + if err != nil { + return emptyStore, fmt.Errorf("unable to configure the storage for %s. Error: %v", kind, err) + } + } + + return NewStowRawStore(fn(cfg.InitContainer), loc, signedURLLoc, cfg.MultiContainerEnabled, metricsScope) } func legacyS3ConfigMap(cfg ConnectionConfig) stow.ConfigMap { diff --git a/flytestdlib/storage/stow_store_test.go b/flytestdlib/storage/stow_store_test.go index c9c022b311..4a446f74de 100644 --- a/flytestdlib/storage/stow_store_test.go +++ b/flytestdlib/storage/stow_store_test.go @@ -173,7 +173,7 @@ func TestStowStore_CreateSignedURL(t *testing.T) { } return nil, fmt.Errorf("container is not supported") }, - }, false, testScope) + }, nil, false, testScope) assert.NoError(t, err) actual, err := s.CreateSignedURL(context.TODO(), DataReference("https://container/path"), SignedURLProperties{}) @@ -197,7 +197,7 @@ func TestStowStore_CreateSignedURL(t *testing.T) { } return nil, fmt.Errorf("container is not supported") }, - }, false, testScope) + }, nil, false, testScope) assert.NoError(t, err) _, err = s.CreateSignedURL(context.TODO(), DataReference("://container/path"), SignedURLProperties{}) @@ -220,7 +220,7 @@ func TestStowStore_CreateSignedURL(t *testing.T) { } return nil, fmt.Errorf("container is not supported") }, - }, false, testScope) + }, nil, false, testScope) assert.NoError(t, err) _, err = s.CreateSignedURL(context.TODO(), DataReference("s3://container2/path"), SignedURLProperties{}) @@ -248,7 +248,7 @@ func TestStowStore_ReadRaw(t *testing.T) { } return nil, fmt.Errorf("container is not supported") }, - }, false, testScope) + }, nil, false, testScope) assert.NoError(t, err) err = s.WriteRaw(context.TODO(), DataReference("s3://container/path"), 0, Options{}, bytes.NewReader([]byte{})) assert.NoError(t, err) @@ -280,7 +280,7 @@ func TestStowStore_ReadRaw(t *testing.T) { } return nil, fmt.Errorf("container is not supported") }, - }, false, testScope) + }, nil, false, testScope) assert.NoError(t, err) err = s.WriteRaw(context.TODO(), DataReference("s3://container/path"), 3*MiB, Options{}, bytes.NewReader([]byte{})) assert.NoError(t, err) @@ -311,7 +311,7 @@ func TestStowStore_ReadRaw(t *testing.T) { } return nil, fmt.Errorf("container is not supported") }, - }, false, testScope) + }, nil, false, testScope) assert.NoError(t, err) err = s.WriteRaw(context.TODO(), DataReference("s3://container/path"), 3*MiB, Options{}, bytes.NewReader([]byte{})) assert.NoError(t, err) @@ -340,7 +340,7 @@ func TestStowStore_ReadRaw(t *testing.T) { } return nil, fmt.Errorf("container is not supported") }, - }, true, testScope) + }, nil, true, testScope) assert.NoError(t, err) err = s.WriteRaw(context.TODO(), "s3://bad-container/path", 0, Options{}, bytes.NewReader([]byte{})) assert.NoError(t, err) @@ -372,7 +372,7 @@ func TestStowStore_ReadRaw(t *testing.T) { } return nil, fmt.Errorf("container is not supported") }, - }, true, testScope) + }, nil, true, testScope) assert.NoError(t, err) err = s.WriteRaw(context.TODO(), "s3://bad-container/path", 0, Options{}, bytes.NewReader([]byte{})) assert.Error(t, err) @@ -618,7 +618,7 @@ func TestStowStore_WriteRaw(t *testing.T) { } return nil, fmt.Errorf("container is not supported") }, - }, true, testScope) + }, nil, true, testScope) assert.NoError(t, err) err = s.WriteRaw(context.TODO(), DataReference("s3://container/path"), 0, Options{}, bytes.NewReader([]byte{})) assert.NoError(t, err) @@ -646,7 +646,7 @@ func TestStowStore_WriteRaw(t *testing.T) { } return nil, fmt.Errorf("container is not supported") }, - }, true, testScope) + }, nil, true, testScope) assert.NoError(t, err) err = s.WriteRaw(context.TODO(), DataReference("s3://container/path"), 0, Options{}, bytes.NewReader([]byte{})) assert.EqualError(t, err, "Failed to write data [0b] to path [path].: foo") diff --git a/flytestdlib/storage/utils.go b/flytestdlib/storage/utils.go index 4c2a7bd729..40da259046 100644 --- a/flytestdlib/storage/utils.go +++ b/flytestdlib/storage/utils.go @@ -63,6 +63,15 @@ func MapStrings(mapper func(string) string, strings ...string) []string { return strings } +// MergeMaps merges all src maps into dst in order. +func MergeMaps(dst map[string]string, src ...map[string]string) { + for _, m := range src { + for k, v := range m { + dst[k] = v + } + } +} + func incFailureCounterForError(ctx context.Context, counter labeled.Counter, err error) { errCode, found := stdErrs.GetErrorCode(err) if found {