diff --git a/generated.go b/generated.go index 7d1031f..761d75f 100644 --- a/generated.go +++ b/generated.go @@ -25,6 +25,8 @@ const Type = "azfile" // ObjectSystemMetadata stores system metadata for object. type ObjectSystemMetadata struct { + // ServerEncrypted + ServerEncrypted bool } // GetObjectSystemMetadata will get ObjectSystemMetadata from Object. @@ -68,27 +70,50 @@ func setStorageSystemMetadata(s *StorageMeta, sm StorageSystemMetadata) { s.SetSystemMetadata(sm) } +// WithDefaultStoragePairs will apply default_storage_pairs value to Options. +// +// DefaultStoragePairs set default pairs for storager actions +func WithDefaultStoragePairs(v DefaultStoragePairs) Pair { + return Pair{ + Key: "default_storage_pairs", + Value: v, + } +} + +// WithStorageFeatures will apply storage_features value to Options. +// +// StorageFeatures set storage features +func WithStorageFeatures(v StorageFeatures) Pair { + return Pair{ + Key: "storage_features", + Value: v, + } +} + var pairMap = map[string]string{ - "content_md5": "string", - "content_type": "string", - "context": "context.Context", - "continuation_token": "string", - "credential": "string", - "endpoint": "string", - "expire": "time.Duration", - "http_client_options": "*httpclient.Options", - "interceptor": "Interceptor", - "io_callback": "func([]byte)", - "list_mode": "ListMode", - "location": "string", - "multipart_id": "string", - "name": "string", - "object_mode": "ObjectMode", - "offset": "int64", - "size": "int64", - "work_dir": "string", + "content_md5": "string", + "content_type": "string", + "context": "context.Context", + "continuation_token": "string", + "credential": "string", + "default_storage_pairs": "DefaultStoragePairs", + "endpoint": "string", + "expire": "time.Duration", + "http_client_options": "*httpclient.Options", + "interceptor": "Interceptor", + "io_callback": "func([]byte)", + "list_mode": "ListMode", + "location": "string", + "multipart_id": "string", + "name": "string", + "object_mode": "ObjectMode", + "offset": "int64", + "size": "int64", + "storage_features": "StorageFeatures", + "work_dir": "string", } var ( + _ Direr = &Storage{} _ Storager = &Storage{} ) @@ -100,7 +125,19 @@ type pairStorageNew struct { pairs []Pair // Required pairs + HasCredential bool + Credential string + HasEndpoint bool + Endpoint string + HasName bool + Name string // Optional pairs + HasDefaultStoragePairs bool + DefaultStoragePairs DefaultStoragePairs + HasStorageFeatures bool + StorageFeatures StorageFeatures + HasWorkDir bool + WorkDir string // Enable features // Default pairs } @@ -114,9 +151,45 @@ func parsePairStorageNew(opts []Pair) (pairStorageNew, error) { for _, v := range opts { switch v.Key { // Required pairs + case "credential": + if result.HasCredential { + continue + } + result.HasCredential = true + result.Credential = v.Value.(string) + case "endpoint": + if result.HasEndpoint { + continue + } + result.HasEndpoint = true + result.Endpoint = v.Value.(string) + case "name": + if result.HasName { + continue + } + result.HasName = true + result.Name = v.Value.(string) // Optional pairs - // Enable features - // Default pairs + case "default_storage_pairs": + if result.HasDefaultStoragePairs { + continue + } + result.HasDefaultStoragePairs = true + result.DefaultStoragePairs = v.Value.(DefaultStoragePairs) + case "storage_features": + if result.HasStorageFeatures { + continue + } + result.HasStorageFeatures = true + result.StorageFeatures = v.Value.(StorageFeatures) + case "work_dir": + if result.HasWorkDir { + continue + } + result.HasWorkDir = true + result.WorkDir = v.Value.(string) + // Enable features + // Default pairs } } @@ -124,18 +197,29 @@ func parsePairStorageNew(opts []Pair) (pairStorageNew, error) { // Default pairs + if !result.HasCredential { + return pairStorageNew{}, services.PairRequiredError{Keys: []string{"credential"}} + } + if !result.HasEndpoint { + return pairStorageNew{}, services.PairRequiredError{Keys: []string{"endpoint"}} + } + if !result.HasName { + return pairStorageNew{}, services.PairRequiredError{Keys: []string{"name"}} + } + return result, nil } // DefaultStoragePairs is default pairs for specific action type DefaultStoragePairs struct { - Create []Pair - Delete []Pair - List []Pair - Metadata []Pair - Read []Pair - Stat []Pair - Write []Pair + Create []Pair + CreateDir []Pair + Delete []Pair + List []Pair + Metadata []Pair + Read []Pair + Stat []Pair + Write []Pair } // pairStorageCreate is the parsed struct @@ -170,6 +254,29 @@ func (s *Storage) parsePairStorageCreate(opts []Pair) (pairStorageCreate, error) return result, nil } +// pairStorageCreateDir is the parsed struct +type pairStorageCreateDir struct { + pairs []Pair +} + +// parsePairStorageCreateDir will parse Pair slice into *pairStorageCreateDir +func (s *Storage) parsePairStorageCreateDir(opts []Pair) (pairStorageCreateDir, error) { + result := pairStorageCreateDir{ + pairs: opts, + } + + for _, v := range opts { + switch v.Key { + default: + return pairStorageCreateDir{}, services.PairUnsupportedError{Pair: v} + } + } + + // Check required pairs. + + return result, nil +} + // pairStorageDelete is the parsed struct type pairStorageDelete struct { pairs []Pair @@ -407,6 +514,31 @@ func (s *Storage) Create(path string, pairs ...Pair) (o *Object) { return s.create(path, opt) } +// CreateDir will create a new dir object. +// +// This function will create a context by default. +func (s *Storage) CreateDir(path string, pairs ...Pair) (o *Object, err error) { + ctx := context.Background() + return s.CreateDirWithContext(ctx, path, pairs...) +} + +// CreateDirWithContext will create a new dir object. +func (s *Storage) CreateDirWithContext(ctx context.Context, path string, pairs ...Pair) (o *Object, err error) { + defer func() { + err = s.formatError("create_dir", err, path) + }() + + pairs = append(pairs, s.defaultPairs.CreateDir...) + var opt pairStorageCreateDir + + opt, err = s.parsePairStorageCreateDir(pairs) + if err != nil { + return + } + + return s.createDir(ctx, path, opt) +} + // Delete will delete an object from service. // // ## Behavior diff --git a/go.mod b/go.mod index 1229aca..407d681 100644 --- a/go.mod +++ b/go.mod @@ -2,4 +2,9 @@ module github.com/beyondstorage/go-service-azfile go 1.15 -require github.com/beyondstorage/go-storage/v4 v4.6.0 +require ( + github.com/Azure/azure-storage-file-go v0.8.0 + github.com/beyondstorage/go-endpoint v1.1.0 + github.com/beyondstorage/go-storage/v4 v4.6.0 + github.com/pkg/errors v0.9.1 // indirect +) diff --git a/go.sum b/go.sum index 1b11c93..8f5037a 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,11 @@ +github.com/Azure/azure-pipeline-go v0.2.1 h1:OLBdZJ3yvOn2MezlWvbrBMTEUQC72zAftRZOMdj5HYo= +github.com/Azure/azure-pipeline-go v0.2.1/go.mod h1:UGSo8XybXnIGZ3epmeBw7Jdz+HiUVpqIlpz/HKHylF4= +github.com/Azure/azure-storage-file-go v0.8.0 h1:OX8DGsleWLUE6Mw4R/OeWEZMvsTIpwN94J59zqKQnTI= +github.com/Azure/azure-storage-file-go v0.8.0/go.mod h1:3w3mufGcMjcOJ3w+4Gs+5wsSgkT7xDwWWqMMIrXtW4c= github.com/Xuanwo/templateutils v0.1.0 h1:WpkWOqQtIQ2vAIpJLa727DdN8WtxhUkkbDGa6UhntJY= github.com/Xuanwo/templateutils v0.1.0/go.mod h1:OdE0DJ+CJxDBq6psX5DPV+gOZi8bhuHuVUpPCG++Wb8= +github.com/beyondstorage/go-endpoint v1.1.0 h1:cpjmQdrAMyaLoT161NIFU/eXcsuMI3xViycid5/mBZg= +github.com/beyondstorage/go-endpoint v1.1.0/go.mod h1:P2hknaGrziOJJKySv/XnAiVw/d3v12/LZu2gSxEx4nM= github.com/beyondstorage/go-storage/v4 v4.6.0 h1:a05dtbYjMZB7LrUSvVzzHwlx33B4yEmd5oQB7Itk7VY= github.com/beyondstorage/go-storage/v4 v4.6.0/go.mod h1:mc9VzBImjXDg1/1sLfta2MJH79elfM6m47ZZvZ+q/Uw= github.com/dave/dst v0.26.2 h1:lnxLAKI3tx7MgLNVDirFCsDTlTG9nKTk7GcptKcWSwY= @@ -11,8 +17,10 @@ github.com/dave/rebecca v0.9.1/go.mod h1:N6XYdMD/OKw3lkF3ywh8Z6wPGuwNFDNtWYEMFWE github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/google/pprof v0.0.0-20181127221834-b4f47329b966/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/kevinburke/go-bindata v3.22.0+incompatible h1:/JmqEhIWQ7GRScV0WjX/0tqBrC5D21ALg0H0U/KZ/ts= @@ -20,10 +28,15 @@ github.com/kevinburke/go-bindata v3.22.0+incompatible/go.mod h1:/pEEZ72flUW2p0yi github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149 h1:HfxbT6/JcvIljmERptWhwa8XzP7H3T+Z2N26gTsaDaA= +github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= github.com/pelletier/go-toml v1.9.3 h1:zeC5b1GviRUyKYd6OJPvBU/mcVDVoL1OhT17FCt5dSQ= github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ= @@ -44,7 +57,9 @@ golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 h1:4nGaVu0QrbjT/AK2PRLuQfQuh6DJve+pELhqTdAj3x0= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -52,6 +67,7 @@ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sys v0.0.0-20180903190138-2b024373dcd9/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -59,6 +75,7 @@ golang.org/x/sys v0.0.0-20210510120138-977fb7262007 h1:gG67DSER+11cZvqIMb8S8bt0v golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= diff --git a/iterator.go b/iterator.go new file mode 100644 index 0000000..ad1e5a9 --- /dev/null +++ b/iterator.go @@ -0,0 +1,16 @@ +package azfile + +import "github.com/Azure/azure-storage-file-go/azfile" + +type objectPageStatus struct { + maxResults int32 + prefix string + marker azfile.Marker +} + +func (i *objectPageStatus) ContinuationToken() string { + if i.marker.NotDone() { + return *i.marker.Val + } + return "" +} diff --git a/service.toml b/service.toml index 5c7ee5a..a552459 100644 --- a/service.toml +++ b/service.toml @@ -1,6 +1,11 @@ name = "azfile" [namespace.storage] +implement = ["direr"] + +[namespace.storage.new] +required = ["name", "credential", "endpoint"] +optional = ["storage_features", "default_storage_pairs", "work_dir"] [namespace.storage.op.create] optional = ["object_mode"] @@ -19,3 +24,14 @@ optional = ["object_mode"] [namespace.storage.op.write] optional = ["content_md5", "content_type", "io_callback"] + +[pairs.storage_features] +type = "StorageFeatures" +description = "set storage features" + +[pairs.default_storage_pairs] +type = "DefaultStoragePairs" +description = "set default pairs for storager actions" + +[infos.object.meta.server-encrypted] +type = "bool" diff --git a/storage.go b/storage.go index 0d49fc8..0c8fe63 100644 --- a/storage.go +++ b/storage.go @@ -2,35 +2,268 @@ package azfile import ( "context" + "encoding/base64" "io" + "strconv" + "github.com/Azure/azure-storage-file-go/azfile" + + "github.com/beyondstorage/go-storage/v4/pkg/iowrap" . "github.com/beyondstorage/go-storage/v4/types" ) func (s *Storage) create(path string, opt pairStorageCreate) (o *Object) { - panic("not implemented") + rp := s.getAbsPath(path) + + if opt.HasObjectMode && opt.ObjectMode.IsDir() { + o = s.newObject(true) + o.Mode |= ModeDir + } else { + o = s.newObject(false) + o.Mode |= ModeRead + } + + o.ID = rp + o.Path = path + + return o +} + +func (s *Storage) createDir(ctx context.Context, path string, opt pairStorageCreateDir) (o *Object, err error) { + rp := s.getAbsPath(path) + + attribute := azfile.FileAttributeNone + + properties := azfile.SMBProperties{ + FileAttributes: &attribute, + } + + fi, err := s.client.NewDirectoryURL(path).GetProperties(ctx) + if err == nil { + // The directory exist, we should set the metadata. + o = s.newObject(true) + o.SetLastModified(fi.LastModified()) + } else if !checkError(err, fileNotFound) { + // Something error other then file not found happened, return directly. + return nil, err + } else { + // The directory not exists, we should create the directory. + _, err = s.client.NewDirectoryURL(path).Create(ctx, nil, properties) + if err != nil { + return nil, err + } + + o = s.newObject(false) + } + + o.ID = rp + o.Path = path + o.Mode |= ModeDir + + return } func (s *Storage) delete(ctx context.Context, path string, opt pairStorageDelete) (err error) { - panic("not implemented") + if opt.HasObjectMode && opt.ObjectMode.IsDir() { + _, err = s.client.NewDirectoryURL(path).Delete(ctx) + } else { + _, err = s.client.NewFileURL(path).Delete(ctx) + } + + if err != nil { + // azfile Delete is not idempotent, so we need to check file not found error. + // + // References + // - [GSP-46](https://github.com/beyondstorage/specs/blob/master/rfcs/46-idempotent-delete.md) + // - https://docs.microsoft.com/en-us/rest/api/storageservices/delete-file2#remarks + if checkError(err, fileNotFound) { + err = nil + } else { + return err + } + } + + return nil } func (s *Storage) list(ctx context.Context, path string, opt pairStorageList) (oi *ObjectIterator, err error) { - panic("not implemented") + input := &objectPageStatus{ + maxResults: 200, + prefix: s.getAbsPath(path), + } + + return NewObjectIterator(ctx, s.nextObjectPage, input), nil } func (s *Storage) metadata(opt pairStorageMetadata) (meta *StorageMeta) { - panic("not implemented") + meta = NewStorageMeta() + meta.WorkDir = s.workDir + return meta +} + +func (s *Storage) nextObjectPage(ctx context.Context, page *ObjectPage) error { + input := page.Status.(*objectPageStatus) + + options := azfile.ListFilesAndDirectoriesOptions{ + Prefix: input.prefix, + MaxResults: input.maxResults, + } + + output, err := s.client.ListFilesAndDirectoriesSegment(ctx, input.marker, options) + if err != nil { + return err + } + + for _, v := range output.DirectoryItems { + o, err := s.formatDirObject(v) + if err != nil { + return err + } + + page.Data = append(page.Data, o) + } + + for _, v := range output.FileItems { + o, err := s.formatFileObject(v) + if err != nil { + return err + } + + page.Data = append(page.Data, o) + } + + if !output.NextMarker.NotDone() { + return IterateDone + } + + input.marker = output.NextMarker + + return nil } func (s *Storage) read(ctx context.Context, path string, w io.Writer, opt pairStorageRead) (n int64, err error) { - panic("not implemented") + offset := int64(0) + if opt.HasOffset { + offset = opt.Offset + } + + count := int64(azfile.CountToEnd) + if opt.HasSize { + count = opt.Size + } + + output, err := s.client.NewFileURL(path).Download(ctx, offset, count, false) + if err != nil { + return 0, err + } + defer func() { + cErr := output.Response().Body.Close() + if cErr != nil { + err = cErr + } + }() + + rc := output.Response().Body + if opt.HasIoCallback { + rc = iowrap.CallbackReadCloser(rc, opt.IoCallback) + } + + return io.Copy(w, rc) } func (s *Storage) stat(ctx context.Context, path string, opt pairStorageStat) (o *Object, err error) { - panic("not implemented") + rp := s.getAbsPath(path) + + var dirOutput *azfile.DirectoryGetPropertiesResponse + var fileOutput *azfile.FileGetPropertiesResponse + + if opt.HasObjectMode && opt.ObjectMode.IsDir() { + dirOutput, err = s.client.NewDirectoryURL(path).GetProperties(ctx) + } else { + fileOutput, err = s.client.NewFileURL(path).GetProperties(ctx) + } + + if err != nil { + return nil, err + } + + o = s.newObject(true) + o.ID = rp + o.Path = path + + if opt.HasObjectMode && opt.ObjectMode.IsDir() { + o.Mode |= ModeDir + + o.SetLastModified(dirOutput.LastModified()) + + if v := string(dirOutput.ETag()); v != "" { + o.SetEtag(v) + } + + var sm ObjectSystemMetadata + if v, err := strconv.ParseBool(dirOutput.IsServerEncrypted()); err == nil { + sm.ServerEncrypted = v + } + o.SetSystemMetadata(sm) + } else { + o.Mode |= ModeRead + + o.SetContentLength(fileOutput.ContentLength()) + o.SetLastModified(fileOutput.LastModified()) + + if v := string(fileOutput.ETag()); v != "" { + o.SetEtag(v) + } + if v := fileOutput.ContentType(); v != "" { + o.SetContentType(v) + } + if v := fileOutput.ContentMD5(); len(v) > 0 { + o.SetContentMd5(base64.StdEncoding.EncodeToString(v)) + } + + var sm ObjectSystemMetadata + if v, err := strconv.ParseBool(fileOutput.IsServerEncrypted()); err == nil { + sm.ServerEncrypted = v + } + o.SetSystemMetadata(sm) + } + + return o, nil } func (s *Storage) write(ctx context.Context, path string, r io.Reader, size int64, opt pairStorageWrite) (n int64, err error) { - panic("not implemented") + if opt.HasIoCallback { + r = iowrap.CallbackReader(r, opt.IoCallback) + } + + headers := azfile.FileHTTPHeaders{} + + if opt.HasContentType { + headers.ContentType = opt.ContentType + } + + // `Create` only initializes the file. + // ref: https://docs.microsoft.com/en-us/rest/api/storageservices/create-file + _, err = s.client.NewFileURL(path).Create(ctx, size, headers, nil) + if err != nil { + return 0, err + } + + body := iowrap.SizedReadSeekCloser(r, size) + + var transactionalMD5 []byte + if opt.HasContentMd5 { + transactionalMD5, err = base64.StdEncoding.DecodeString(opt.ContentMd5) + if err != nil { + return 0, err + } + } + + // Since `Create' only initializes the file, we need to call `UploadRange' to write the contents to the file. + _, err = s.client.NewFileURL(path).UploadRange(ctx, 0, body, transactionalMD5) + if err != nil { + return 0, err + } + + return size, nil } diff --git a/utils.go b/utils.go index fb81645..6cc550b 100644 --- a/utils.go +++ b/utils.go @@ -1,21 +1,36 @@ package azfile import ( + "fmt" + "net/url" + "strings" + "time" + + "github.com/Azure/azure-storage-file-go/azfile" + + "github.com/beyondstorage/go-endpoint" + ps "github.com/beyondstorage/go-storage/v4/pairs" + "github.com/beyondstorage/go-storage/v4/pkg/credential" "github.com/beyondstorage/go-storage/v4/services" "github.com/beyondstorage/go-storage/v4/types" ) -// Storage is the example client. +// Storage is the azfile client. type Storage struct { + client azfile.DirectoryURL + + workDir string + defaultPairs DefaultStoragePairs features StorageFeatures types.UnimplementedStorager + types.UnimplementedDirer } // String implements Storager.String func (s *Storage) String() string { - panic("implement me") + return fmt.Sprintf("Storager azfile {WorkDir: %s}", s.workDir) } // NewStorager will create Storager only. @@ -31,7 +46,77 @@ func newStorager(pairs ...types.Pair) (store *Storage, err error) { } }() - panic("implement me") + opt, err := parsePairStorageNew(pairs) + if err != nil { + return nil, err + } + + store = &Storage{ + workDir: "/", + } + + if opt.HasWorkDir { + store.workDir = opt.WorkDir + } + + ep, err := endpoint.Parse(opt.Endpoint) + if err != nil { + return nil, err + } + + var uri string + switch ep.Protocol() { + case endpoint.ProtocolHTTP: + uri, _, _ = ep.HTTP() + case endpoint.ProtocolHTTPS: + uri, _, _ = ep.HTTPS() + default: + return nil, services.PairUnsupportedError{Pair: ps.WithEndpoint(opt.Endpoint)} + } + + if strings.HasPrefix(store.workDir, "/") { + uri = fmt.Sprintln(uri + store.workDir) + } else { + uri = fmt.Sprintln(uri + "/" + store.workDir) + } + + primaryURL, _ := url.Parse(uri) + + cred, err := credential.Parse(opt.Credential) + if err != nil { + return nil, err + } + if cred.Protocol() != credential.ProtocolHmac { + return nil, services.PairUnsupportedError{Pair: ps.WithCredential(opt.Credential)} + } + + credValue, err := azfile.NewSharedKeyCredential(cred.Hmac()) + if err != nil { + return nil, err + } + + p := azfile.NewPipeline(credValue, azfile.PipelineOptions{ + Retry: azfile.RetryOptions{ + // Use a fixed back-off retry policy. + Policy: 1, + // A value of 1 means 1 try and no retries. + MaxTries: 1, + // Set a long enough timeout to adopt our timeout control. + // This value could be adjusted to context deadline if request context has a deadline set. + TryTimeout: 720 * time.Hour, + }, + }) + + store.client = azfile.NewDirectoryURL(*primaryURL, p) + + if opt.HasDefaultStoragePairs { + store.defaultPairs = opt.DefaultStoragePairs + } + if opt.HasStorageFeatures { + store.features = opt.StorageFeatures + } + + return store, nil } func (s *Storage) formatError(op string, err error, path ...string) error { @@ -54,5 +139,77 @@ func formatError(err error) error { return err } - panic("implement me") + e, ok := err.(azfile.StorageError) + + if ok { + switch azfile.StorageErrorCodeType(e.ServiceCode()) { + case "": + switch e.Response().StatusCode { + case fileNotFound: + return fmt.Errorf("%w: %v", services.ErrObjectNotExist, err) + default: + return fmt.Errorf("%w: %v", services.ErrUnexpected, err) + } + case azfile.StorageErrorCodeResourceNotFound: + return fmt.Errorf("%w: %v", services.ErrObjectNotExist, err) + case azfile.StorageErrorCodeInsufficientAccountPermissions: + return fmt.Errorf("%w: %v", services.ErrPermissionDenied, err) + default: + return fmt.Errorf("%w: %v", services.ErrUnexpected, err) + } + } + + return fmt.Errorf("%w: %v", services.ErrUnexpected, err) +} + +// getAbsPath will calculate object storage's abs path +func (s *Storage) getAbsPath(path string) string { + prefix := strings.TrimPrefix(s.workDir, "/") + return prefix + path +} + +// getRelPath will get object storage's rel path. +func (s *Storage) getRelPath(path string) string { + prefix := strings.TrimPrefix(s.workDir, "/") + return strings.TrimPrefix(path, prefix) +} + +func (s *Storage) newObject(done bool) *types.Object { + return types.NewObject(s, done) +} + +func (s *Storage) formatFileObject(v azfile.FileItem) (o *types.Object, err error) { + o = s.newObject(true) + o.ID = v.Name + o.Path = s.getRelPath(v.Name) + o.Mode |= types.ModeRead + + if v.Properties.ContentLength != 0 { + o.SetContentLength(v.Properties.ContentLength) + } + + return +} + +func (s *Storage) formatDirObject(v azfile.DirectoryItem) (o *types.Object, err error) { + o = s.newObject(true) + o.ID = v.Name + o.Path = s.getRelPath(v.Name) + o.Mode |= types.ModeDir + + return +} + +const ( + // File not found error. + fileNotFound = 404 +) + +func checkError(err error, expect int) bool { + e, ok := err.(azfile.StorageError) + if !ok { + return false + } + + return e.Response().StatusCode == expect }