diff --git a/generated.go b/generated.go index 499974d..586764a 100644 --- a/generated.go +++ b/generated.go @@ -210,6 +210,8 @@ var pairMap = map[string]string{ "copy_source_encryption_customer_algorithm": "string", "copy_source_encryption_customer_key": "[]byte", "credential": "string", + "default_content_type": "string", + "default_io_callback": "func([]byte)", "default_service_pairs": "DefaultServicePairs", "default_storage_pairs": "DefaultStoragePairs", "disable_uri_cleaning": "bool", @@ -558,15 +560,16 @@ func (s *Service) ListWithContext(ctx context.Context, pairs ...Pair) (sti *Stor } var ( - _ Appender = &Storage{} - _ Copier = &Storage{} - _ Direr = &Storage{} - _ Fetcher = &Storage{} - _ Linker = &Storage{} - _ Mover = &Storage{} - _ Multiparter = &Storage{} - _ Reacher = &Storage{} - _ Storager = &Storage{} + _ Appender = &Storage{} + _ Copier = &Storage{} + _ Direr = &Storage{} + _ Fetcher = &Storage{} + _ Linker = &Storage{} + _ Mover = &Storage{} + _ Multiparter = &Storage{} + _ Reacher = &Storage{} + _ StorageHTTPSigner = &Storage{} + _ Storager = &Storage{} ) type StorageFeatures struct { @@ -612,6 +615,10 @@ type pairStorageNew struct { hasEnableVirtualLink bool EnableVirtualLink bool // Default pairs + hasDefaultContentType bool + DefaultContentType string + hasDefaultIoCallback bool + DefaultIoCallback func([]byte) } // parsePairStorageNew will parse Pair slice into *pairStorageNew @@ -679,7 +686,19 @@ func parsePairStorageNew(opts []Pair) (pairStorageNew, error) { } result.hasEnableVirtualLink = true result.EnableVirtualLink = true - // Default pairs + // Default pairs + case "default_content_type": + if result.hasDefaultContentType { + continue + } + result.hasDefaultContentType = true + result.DefaultContentType = v.Value.(string) + case "default_io_callback": + if result.hasDefaultIoCallback { + continue + } + result.hasDefaultIoCallback = true + result.DefaultIoCallback = v.Value.(func([]byte)) } } @@ -694,6 +713,18 @@ func parsePairStorageNew(opts []Pair) (pairStorageNew, error) { } // Default pairs + if result.hasDefaultContentType { + result.HasDefaultStoragePairs = true + result.DefaultStoragePairs.CreateAppend = append(result.DefaultStoragePairs.CreateAppend, WithContentType(result.DefaultContentType)) + result.DefaultStoragePairs.QuerySignHTTPWrite = append(result.DefaultStoragePairs.QuerySignHTTPWrite, WithContentType(result.DefaultContentType)) + result.DefaultStoragePairs.Write = append(result.DefaultStoragePairs.Write, WithContentType(result.DefaultContentType)) + } + if result.hasDefaultIoCallback { + result.HasDefaultStoragePairs = true + result.DefaultStoragePairs.Read = append(result.DefaultStoragePairs.Read, WithIoCallback(result.DefaultIoCallback)) + result.DefaultStoragePairs.Write = append(result.DefaultStoragePairs.Write, WithIoCallback(result.DefaultIoCallback)) + result.DefaultStoragePairs.WriteMultipart = append(result.DefaultStoragePairs.WriteMultipart, WithIoCallback(result.DefaultIoCallback)) + } if !result.HasName { return pairStorageNew{}, services.PairRequiredError{Keys: []string{"name"}} @@ -704,26 +735,28 @@ func parsePairStorageNew(opts []Pair) (pairStorageNew, error) { // DefaultStoragePairs is default pairs for specific action type DefaultStoragePairs struct { - CommitAppend []Pair - CompleteMultipart []Pair - Copy []Pair - Create []Pair - CreateAppend []Pair - CreateDir []Pair - CreateLink []Pair - CreateMultipart []Pair - Delete []Pair - Fetch []Pair - List []Pair - ListMultipart []Pair - Metadata []Pair - Move []Pair - Reach []Pair - Read []Pair - Stat []Pair - Write []Pair - WriteAppend []Pair - WriteMultipart []Pair + CommitAppend []Pair + CompleteMultipart []Pair + Copy []Pair + Create []Pair + CreateAppend []Pair + CreateDir []Pair + CreateLink []Pair + CreateMultipart []Pair + Delete []Pair + Fetch []Pair + List []Pair + ListMultipart []Pair + Metadata []Pair + Move []Pair + QuerySignHTTPRead []Pair + QuerySignHTTPWrite []Pair + Reach []Pair + Read []Pair + Stat []Pair + Write []Pair + WriteAppend []Pair + WriteMultipart []Pair } // pairStorageCommitAppend is the parsed struct @@ -1174,6 +1207,133 @@ func (s *Storage) parsePairStorageMove(opts []Pair) (pairStorageMove, error) { return result, nil } +// pairStorageQuerySignHTTPRead is the parsed struct +type pairStorageQuerySignHTTPRead struct { + pairs []Pair + HasEncryptionCustomerAlgorithm bool + EncryptionCustomerAlgorithm string + HasEncryptionCustomerKey bool + EncryptionCustomerKey []byte + HasOffset bool + Offset int64 + HasSize bool + Size int64 +} + +// parsePairStorageQuerySignHTTPRead will parse Pair slice into *pairStorageQuerySignHTTPRead +func (s *Storage) parsePairStorageQuerySignHTTPRead(opts []Pair) (pairStorageQuerySignHTTPRead, error) { + result := pairStorageQuerySignHTTPRead{ + pairs: opts, + } + + for _, v := range opts { + switch v.Key { + case "encryption_customer_algorithm": + if result.HasEncryptionCustomerAlgorithm { + continue + } + result.HasEncryptionCustomerAlgorithm = true + result.EncryptionCustomerAlgorithm = v.Value.(string) + continue + case "encryption_customer_key": + if result.HasEncryptionCustomerKey { + continue + } + result.HasEncryptionCustomerKey = true + result.EncryptionCustomerKey = v.Value.([]byte) + continue + case "offset": + if result.HasOffset { + continue + } + result.HasOffset = true + result.Offset = v.Value.(int64) + continue + case "size": + if result.HasSize { + continue + } + result.HasSize = true + result.Size = v.Value.(int64) + continue + default: + return pairStorageQuerySignHTTPRead{}, services.PairUnsupportedError{Pair: v} + } + } + + // Check required pairs. + + return result, nil +} + +// pairStorageQuerySignHTTPWrite is the parsed struct +type pairStorageQuerySignHTTPWrite struct { + pairs []Pair + HasContentMd5 bool + ContentMd5 string + HasContentType bool + ContentType string + HasEncryptionCustomerAlgorithm bool + EncryptionCustomerAlgorithm string + HasEncryptionCustomerKey bool + EncryptionCustomerKey []byte + HasStorageClass bool + StorageClass string +} + +// parsePairStorageQuerySignHTTPWrite will parse Pair slice into *pairStorageQuerySignHTTPWrite +func (s *Storage) parsePairStorageQuerySignHTTPWrite(opts []Pair) (pairStorageQuerySignHTTPWrite, error) { + result := pairStorageQuerySignHTTPWrite{ + pairs: opts, + } + + for _, v := range opts { + switch v.Key { + case "content_md5": + if result.HasContentMd5 { + continue + } + result.HasContentMd5 = true + result.ContentMd5 = v.Value.(string) + continue + case "content_type": + if result.HasContentType { + continue + } + result.HasContentType = true + result.ContentType = v.Value.(string) + continue + case "encryption_customer_algorithm": + if result.HasEncryptionCustomerAlgorithm { + continue + } + result.HasEncryptionCustomerAlgorithm = true + result.EncryptionCustomerAlgorithm = v.Value.(string) + continue + case "encryption_customer_key": + if result.HasEncryptionCustomerKey { + continue + } + result.HasEncryptionCustomerKey = true + result.EncryptionCustomerKey = v.Value.([]byte) + continue + case "storage_class": + if result.HasStorageClass { + continue + } + result.HasStorageClass = true + result.StorageClass = v.Value.(string) + continue + default: + return pairStorageQuerySignHTTPWrite{}, services.PairUnsupportedError{Pair: v} + } + } + + // Check required pairs. + + return result, nil +} + // pairStorageReach is the parsed struct type pairStorageReach struct { pairs []Pair @@ -1948,8 +2108,60 @@ func (s *Storage) MoveWithContext(ctx context.Context, src string, dst string, p return s.move(ctx, src, dst, opt) } +// QuerySignHTTPRead will read data from the file by using query parameters to authenticate requests. +// +// This function will create a context by default. +func (s *Storage) QuerySignHTTPRead(path string, expire time.Duration, pairs ...Pair) (req *http.Request, err error) { + ctx := context.Background() + return s.QuerySignHTTPReadWithContext(ctx, path, expire, pairs...) +} + +// QuerySignHTTPReadWithContext will read data from the file by using query parameters to authenticate requests. +func (s *Storage) QuerySignHTTPReadWithContext(ctx context.Context, path string, expire time.Duration, pairs ...Pair) (req *http.Request, err error) { + defer func() { + err = s.formatError("query_sign_http_read", err, path) + }() + + pairs = append(pairs, s.defaultPairs.QuerySignHTTPRead...) + var opt pairStorageQuerySignHTTPRead + + opt, err = s.parsePairStorageQuerySignHTTPRead(pairs) + if err != nil { + return + } + + return s.querySignHTTPRead(ctx, path, expire, opt) +} + +// QuerySignHTTPWrite will write data into a file by using query parameters to authenticate requests. +// +// This function will create a context by default. +func (s *Storage) QuerySignHTTPWrite(path string, size int64, expire time.Duration, pairs ...Pair) (req *http.Request, err error) { + ctx := context.Background() + return s.QuerySignHTTPWriteWithContext(ctx, path, size, expire, pairs...) +} + +// QuerySignHTTPWriteWithContext will write data into a file by using query parameters to authenticate requests. +func (s *Storage) QuerySignHTTPWriteWithContext(ctx context.Context, path string, size int64, expire time.Duration, pairs ...Pair) (req *http.Request, err error) { + defer func() { + err = s.formatError("query_sign_http_write", err, path) + }() + + pairs = append(pairs, s.defaultPairs.QuerySignHTTPWrite...) + var opt pairStorageQuerySignHTTPWrite + + opt, err = s.parsePairStorageQuerySignHTTPWrite(pairs) + if err != nil { + return + } + + return s.querySignHTTPWrite(ctx, path, size, expire, opt) +} + // Reach will provide a way, which can reach the object. // +// Deprecated: Use QuerySignHTTPRead instead. +// // This function will create a context by default. func (s *Storage) Reach(path string, pairs ...Pair) (url string, err error) { ctx := context.Background() @@ -1957,6 +2169,8 @@ func (s *Storage) Reach(path string, pairs ...Pair) (url string, err error) { } // ReachWithContext will provide a way, which can reach the object. +// +// Deprecated: Use QuerySignHTTPRead instead. func (s *Storage) ReachWithContext(ctx context.Context, path string, pairs ...Pair) (url string, err error) { defer func() { err = s.formatError("reach", err, path) diff --git a/go.mod b/go.mod index d73e581..30e3e81 100644 --- a/go.mod +++ b/go.mod @@ -5,8 +5,8 @@ go 1.14 require ( bou.ke/monkey v1.0.2 github.com/beyondstorage/go-endpoint v1.1.0 - github.com/beyondstorage/go-integration-test/v4 v4.3.0 - github.com/beyondstorage/go-storage/v4 v4.6.0 + github.com/beyondstorage/go-integration-test/v4 v4.4.0 + github.com/beyondstorage/go-storage/v4 v4.7.0 github.com/golang/mock v1.6.0 github.com/google/uuid v1.3.0 github.com/pengsrc/go-shared v0.2.1-0.20190131101655-1999055a4a14 diff --git a/go.sum b/go.sum index f4f7938..f1efb0c 100644 --- a/go.sum +++ b/go.sum @@ -6,11 +6,10 @@ github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLj github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beyondstorage/go-endpoint v1.1.0 h1:cpjmQdrAMyaLoT161NIFU/eXcsuMI3xViycid5/mBZg= github.com/beyondstorage/go-endpoint v1.1.0/go.mod h1:P2hknaGrziOJJKySv/XnAiVw/d3v12/LZu2gSxEx4nM= -github.com/beyondstorage/go-integration-test/v4 v4.3.0 h1:WZ95f78RKlHpvft8zHcMaoa2aaTF/jzlzINhMD0EMHY= -github.com/beyondstorage/go-integration-test/v4 v4.3.0/go.mod h1:HKgzemQZpxoHBL49JYEUnLTb5eteUhzcvmmPL7EDT/Y= -github.com/beyondstorage/go-storage/v4 v4.4.0/go.mod h1:mc9VzBImjXDg1/1sLfta2MJH79elfM6m47ZZvZ+q/Uw= -github.com/beyondstorage/go-storage/v4 v4.6.0 h1:a05dtbYjMZB7LrUSvVzzHwlx33B4yEmd5oQB7Itk7VY= -github.com/beyondstorage/go-storage/v4 v4.6.0/go.mod h1:mc9VzBImjXDg1/1sLfta2MJH79elfM6m47ZZvZ+q/Uw= +github.com/beyondstorage/go-integration-test/v4 v4.4.0 h1:wiItWmhoAY71Fp76u6u95jJn5m6swM9N/xNV9mW7nyI= +github.com/beyondstorage/go-integration-test/v4 v4.4.0/go.mod h1:o0pHhyaRR/OO6QxnRqWW3aFjTycBZjoKSdHKN/JZRjo= +github.com/beyondstorage/go-storage/v4 v4.7.0 h1:7hpRFNoPY0vWRSH/p2biD2dUQdZMrs1TuIkkqDefmCE= +github.com/beyondstorage/go-storage/v4 v4.7.0/go.mod h1:mc9VzBImjXDg1/1sLfta2MJH79elfM6m47ZZvZ+q/Uw= github.com/dave/dst v0.26.2 h1:lnxLAKI3tx7MgLNVDirFCsDTlTG9nKTk7GcptKcWSwY= github.com/dave/dst v0.26.2/go.mod h1:UMDJuIRPfyUCC78eFuB+SV/WI8oDeyFDvM/JR6NI3IU= github.com/dave/gopackages v0.0.0-20170318123100-46e7023ec56e/go.mod h1:i00+b/gKdIDIxuLDFob7ustLAVqhsZRk2qVZrArELGQ= diff --git a/service.toml b/service.toml index 9ca069f..6a07417 100644 --- a/service.toml +++ b/service.toml @@ -18,7 +18,7 @@ optional = ["location"] [namespace.storage] features = ["virtual_dir", "virtual_link"] -implement = ["appender", "copier", "direr", "fetcher", "mover", "multiparter", "reacher", "linker"] +implement = ["appender", "copier", "direr", "fetcher", "linker", "mover", "multiparter", "reacher", "storage_http_signer"] [namespace.storage.new] required = ["name"] @@ -63,6 +63,12 @@ optional = ["encryption_customer_algorithm", "encryption_customer_key"] [namespace.storage.op.write_multipart] optional = ["encryption_customer_algorithm", "encryption_customer_key", "io_callback"] +[namespace.storage.op.query_sign_http_read] +optional = ["offset", "encryption_customer_algorithm", "encryption_customer_key", "size"] + +[namespace.storage.op.query_sign_http_write] +optional = ["content_md5", "content_type", "encryption_customer_algorithm", "encryption_customer_key", "storage_class"] + [pairs.service_features] type = "ServiceFeatures" description = "set service features" diff --git a/storage.go b/storage.go index faa168f..6bd7ff9 100644 --- a/storage.go +++ b/storage.go @@ -4,13 +4,14 @@ import ( "context" "fmt" "io" + "net/http" "net/url" + "time" "github.com/pengsrc/go-shared/convert" "github.com/qingstor/qingstor-sdk-go/v4/service" ps "github.com/beyondstorage/go-storage/v4/pairs" - "github.com/beyondstorage/go-storage/v4/pkg/headers" "github.com/beyondstorage/go-storage/v4/pkg/iowrap" "github.com/beyondstorage/go-storage/v4/services" . "github.com/beyondstorage/go-storage/v4/types" @@ -515,6 +516,66 @@ func (s *Storage) nextPartPage(ctx context.Context, page *PartPage) error { return nil } +func (s *Storage) querySignHTTPRead(ctx context.Context, path string, expire time.Duration, opt pairStorageQuerySignHTTPRead) (req *http.Request, err error) { + pairs, err := s.parsePairStorageRead(opt.pairs) + if err != nil { + return + } + + input, err := s.formatGetObjectInput(pairs) + if err != nil { + return + } + + bucket := s.bucket.(*service.Bucket) + + rp := s.getAbsPath(path) + + r, _, err := bucket.GetObjectRequest(rp, input) + if err != nil { + return + } + if err = r.BuildWithContext(ctx); err != nil { + return + } + + if err = r.SignQuery(int(expire.Seconds())); err != nil { + return + } + + return r.HTTPRequest, nil +} + +func (s *Storage) querySignHTTPWrite(ctx context.Context, path string, size int64, expire time.Duration, opt pairStorageQuerySignHTTPWrite) (req *http.Request, err error) { + pairs, err := s.parsePairStorageWrite(opt.pairs) + if err != nil { + return + } + + input, err := s.formatPutObjectInput(size, pairs) + if err != nil { + return + } + + bucket := s.bucket.(*service.Bucket) + + rp := s.getAbsPath(path) + + r, _, err := bucket.PutObjectRequest(rp, input) + if err != nil { + return + } + if err = r.BuildWithContext(ctx); err != nil { + return + } + + if err = r.SignQuery(int(expire.Seconds())); err != nil { + return + } + + return r.HTTPRequest, nil +} + func (s *Storage) reach(ctx context.Context, path string, opt pairStorageReach) (url string, err error) { // FIXME: sdk should export GetObjectRequest as interface too? bucket := s.bucket.(*service.Bucket) @@ -537,17 +598,9 @@ func (s *Storage) reach(ctx context.Context, path string, opt pairStorageReach) } func (s *Storage) read(ctx context.Context, path string, w io.Writer, opt pairStorageRead) (n int64, err error) { - input := &service.GetObjectInput{} - if opt.HasEncryptionCustomerAlgorithm { - input.XQSEncryptionCustomerAlgorithm, input.XQSEncryptionCustomerKey, input.XQSEncryptionCustomerKeyMD5, err = calculateEncryptionHeaders(opt.EncryptionCustomerAlgorithm, opt.EncryptionCustomerKey) - if err != nil { - return - } - } - - if opt.HasOffset || opt.HasSize { - rs := headers.FormatRange(opt.Offset, opt.Size) - input.Range = &rs + input, err := s.formatGetObjectInput(opt) + if err != nil { + return } rp := s.getAbsPath(path) @@ -664,22 +717,11 @@ func (s *Storage) write(ctx context.Context, path string, r io.Reader, size int6 r = iowrap.CallbackReader(r, opt.IoCallback) } - input := &service.PutObjectInput{ - ContentLength: &size, - Body: io.LimitReader(r, size), - } - if opt.HasContentMd5 { - input.ContentMD5 = &opt.ContentMd5 - } - if opt.HasStorageClass { - input.XQSStorageClass = service.String(opt.StorageClass) - } - if opt.HasEncryptionCustomerAlgorithm { - input.XQSEncryptionCustomerAlgorithm, input.XQSEncryptionCustomerKey, input.XQSEncryptionCustomerKeyMD5, err = calculateEncryptionHeaders(opt.EncryptionCustomerAlgorithm, opt.EncryptionCustomerKey) - if err != nil { - return - } + input, err := s.formatPutObjectInput(size, opt) + if err != nil { + return } + input.Body = io.LimitReader(r, size) rp := s.getAbsPath(path) diff --git a/tests/storage_test.go b/tests/storage_test.go index 52309db..121c322 100644 --- a/tests/storage_test.go +++ b/tests/storage_test.go @@ -41,3 +41,11 @@ func TestLinker(t *testing.T) { } tests.TestLinker(t, setupTest(t)) } + +func TestSigner(t *testing.T) { + if os.Getenv("STORAGE_QINGSTOR_INTEGRATION_TEST") != "on" { + t.Skipf("STORAGE_QINGSTOR_INTEGRATION_TEST is not 'on', skipped") + } + tests.TestStorageHTTPSignerRead(t, setupTest(t)) + tests.TestStorageHTTPSignerWrite(t, setupTest(t)) +} diff --git a/utils.go b/utils.go index f4477dd..1dd6026 100644 --- a/utils.go +++ b/utils.go @@ -68,6 +68,7 @@ type Storage struct { typ.UnimplementedAppender typ.UnimplementedDirer typ.UnimplementedLinker + typ.UnimplementedStorageHTTPSigner } // String implements Storager.String @@ -437,3 +438,40 @@ func calculateEncryptionHeaders(algo string, key []byte) (algorithm, keyBase64, kMD5B64 := base64.StdEncoding.EncodeToString(kMD5[:]) return &algo, &kB64, &kMD5B64, nil } + +func (s *Storage) formatGetObjectInput(opt pairStorageRead) (input *service.GetObjectInput, err error) { + input = &service.GetObjectInput{} + if opt.HasEncryptionCustomerAlgorithm { + input.XQSEncryptionCustomerAlgorithm, input.XQSEncryptionCustomerKey, input.XQSEncryptionCustomerKeyMD5, err = calculateEncryptionHeaders(opt.EncryptionCustomerAlgorithm, opt.EncryptionCustomerKey) + if err != nil { + return + } + } + + if opt.HasOffset || opt.HasSize { + rs := headers.FormatRange(opt.Offset, opt.Size) + input.Range = &rs + } + + return +} + +func (s *Storage) formatPutObjectInput(size int64, opt pairStorageWrite) (input *service.PutObjectInput, err error) { + input = &service.PutObjectInput{ + ContentLength: &size, + } + if opt.HasContentMd5 { + input.ContentMD5 = service.String(opt.ContentMd5) + } + if opt.HasStorageClass { + input.XQSStorageClass = service.String(opt.StorageClass) + } + if opt.HasEncryptionCustomerAlgorithm { + input.XQSEncryptionCustomerAlgorithm, input.XQSEncryptionCustomerKey, input.XQSEncryptionCustomerKeyMD5, err = calculateEncryptionHeaders(opt.EncryptionCustomerAlgorithm, opt.EncryptionCustomerKey) + if err != nil { + return + } + } + + return +}