From 939768a1af06cf2064761353685f4f090d699c8b Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Fri, 3 Dec 2021 23:57:13 +0100 Subject: [PATCH 01/58] [Build-deps] Additional rules for CODEOWNERS (#2323) --- CODEOWNERS | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CODEOWNERS b/CODEOWNERS index 830c037728..4c9822be14 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -11,5 +11,7 @@ pkg/storage/fs/owncloudsql @cs3org/owncloud-team pkg/storage/fs/owncloud @cs3org/owncloud-team pkg/storage/fs/ocis @cs3org/owncloud-team pkg/storage/utils/decomposedfs @cs3org/owncloud-team +pkg/storage/utils/eosfs @labkode @ishank011 @glpatcern pkg/user/manager/owncloudsql @cs3org/owncloud-team +pkg/eosclient @labkode @ishank011 @glpatcern pkg/app @labkode @ishank011 @wkloucek @glpatcern From ed9517bc36eaaed3b448ca4fb3aca1c5e0df27cb Mon Sep 17 00:00:00 2001 From: Gianmaria Del Monte <39946305+gmgigi96@users.noreply.github.com> Date: Mon, 6 Dec 2021 11:07:24 +0100 Subject: [PATCH 02/58] Remove share refs from trashbin (#2298) --- changelog/unreleased/fix-share-recyclebin.md | 3 +++ pkg/eosclient/eosbinary/eosbinary.go | 8 ++++++-- pkg/eosclient/eosclient.go | 2 +- pkg/eosclient/eosgrpc/eosgrpc.go | 13 +++++++------ pkg/storage/utils/eosfs/eosfs.go | 14 ++++++-------- 5 files changed, 23 insertions(+), 17 deletions(-) create mode 100644 changelog/unreleased/fix-share-recyclebin.md diff --git a/changelog/unreleased/fix-share-recyclebin.md b/changelog/unreleased/fix-share-recyclebin.md new file mode 100644 index 0000000000..fb0b500138 --- /dev/null +++ b/changelog/unreleased/fix-share-recyclebin.md @@ -0,0 +1,3 @@ +Bugfix: Remove share refs from trashbin + +https://github.com/cs3org/reva/pull/2298 \ No newline at end of file diff --git a/pkg/eosclient/eosbinary/eosbinary.go b/pkg/eosclient/eosbinary/eosbinary.go index 6773513962..b9c1f6a5a4 100644 --- a/pkg/eosclient/eosbinary/eosbinary.go +++ b/pkg/eosclient/eosbinary/eosbinary.go @@ -632,8 +632,12 @@ func (c *Client) CreateDir(ctx context.Context, auth eosclient.Authorization, pa } // Remove removes the resource at the given path -func (c *Client) Remove(ctx context.Context, auth eosclient.Authorization, path string) error { - args := []string{"rm", "-r", path} +func (c *Client) Remove(ctx context.Context, auth eosclient.Authorization, path string, noRecycle bool) error { + args := []string{"rm", "-r"} + if noRecycle { + args = append(args, "--no-recycle-bin") // do not put the file in the recycle bin + } + args = append(args, path) _, _, err := c.executeEOS(ctx, args, auth) return err } diff --git a/pkg/eosclient/eosclient.go b/pkg/eosclient/eosclient.go index 30aed21697..c4e47e7b23 100644 --- a/pkg/eosclient/eosclient.go +++ b/pkg/eosclient/eosclient.go @@ -44,7 +44,7 @@ type EOSClient interface { Chown(ctx context.Context, auth, chownauth Authorization, path string) error Chmod(ctx context.Context, auth Authorization, mode, path string) error CreateDir(ctx context.Context, auth Authorization, path string) error - Remove(ctx context.Context, auth Authorization, path string) error + Remove(ctx context.Context, auth Authorization, path string, noRecycle bool) error Rename(ctx context.Context, auth Authorization, oldPath, newPath string) error List(ctx context.Context, auth Authorization, path string) ([]*FileInfo, error) Read(ctx context.Context, auth Authorization, path string) (io.ReadCloser, error) diff --git a/pkg/eosclient/eosgrpc/eosgrpc.go b/pkg/eosclient/eosgrpc/eosgrpc.go index 5e4df305d0..ea906c9de5 100644 --- a/pkg/eosclient/eosgrpc/eosgrpc.go +++ b/pkg/eosclient/eosgrpc/eosgrpc.go @@ -981,7 +981,7 @@ func (c *Client) CreateDir(ctx context.Context, auth eosclient.Authorization, pa } -func (c *Client) rm(ctx context.Context, auth eosclient.Authorization, path string) error { +func (c *Client) rm(ctx context.Context, auth eosclient.Authorization, path string, noRecycle bool) error { log := appctx.GetLogger(ctx) log.Info().Str("func", "rm").Str("uid,gid", auth.Role.UID+","+auth.Role.GID).Str("path", path).Msg("") @@ -995,6 +995,7 @@ func (c *Client) rm(ctx context.Context, auth eosclient.Authorization, path stri msg.Id = new(erpc.MDId) msg.Id.Path = []byte(path) + msg.Norecycle = noRecycle rq.Command = &erpc.NSRequest_Unlink{Unlink: msg} @@ -1016,7 +1017,7 @@ func (c *Client) rm(ctx context.Context, auth eosclient.Authorization, path stri } -func (c *Client) rmdir(ctx context.Context, auth eosclient.Authorization, path string) error { +func (c *Client) rmdir(ctx context.Context, auth eosclient.Authorization, path string, noRecycle bool) error { log := appctx.GetLogger(ctx) log.Info().Str("func", "rmdir").Str("uid,gid", auth.Role.UID+","+auth.Role.GID).Str("path", path).Msg("") @@ -1031,7 +1032,7 @@ func (c *Client) rmdir(ctx context.Context, auth eosclient.Authorization, path s msg.Id = new(erpc.MDId) msg.Id.Path = []byte(path) msg.Recursive = true - msg.Norecycle = false + msg.Norecycle = noRecycle rq.Command = &erpc.NSRequest_Rm{Rm: msg} @@ -1053,7 +1054,7 @@ func (c *Client) rmdir(ctx context.Context, auth eosclient.Authorization, path s } // Remove removes the resource at the given path -func (c *Client) Remove(ctx context.Context, auth eosclient.Authorization, path string) error { +func (c *Client) Remove(ctx context.Context, auth eosclient.Authorization, path string, noRecycle bool) error { log := appctx.GetLogger(ctx) log.Info().Str("func", "Remove").Str("uid,gid", auth.Role.UID+","+auth.Role.GID).Str("path", path).Msg("") @@ -1064,10 +1065,10 @@ func (c *Client) Remove(ctx context.Context, auth eosclient.Authorization, path } if nfo.IsDir { - return c.rmdir(ctx, auth, path) + return c.rmdir(ctx, auth, path, noRecycle) } - return c.rm(ctx, auth, path) + return c.rm(ctx, auth, path, noRecycle) } // Rename renames the resource referenced by oldPath to newPath diff --git a/pkg/storage/utils/eosfs/eosfs.go b/pkg/storage/utils/eosfs/eosfs.go index 602496a2c5..3d045dd553 100644 --- a/pkg/storage/utils/eosfs/eosfs.go +++ b/pkg/storage/utils/eosfs/eosfs.go @@ -1283,7 +1283,7 @@ func (fs *eosfs) Delete(ctx context.Context, ref *provider.Reference) error { return err } - return fs.c.Remove(ctx, auth, fn) + return fs.c.Remove(ctx, auth, fn, false) } func (fs *eosfs) deleteShadow(ctx context.Context, p string) error { @@ -1292,19 +1292,17 @@ func (fs *eosfs) deleteShadow(ctx context.Context, p string) error { } if fs.isShareFolderChild(ctx, p) { - u, err := getUser(ctx) - if err != nil { - return errors.Wrap(err, "eosfs: no user in ctx") - } - fn := fs.wrapShadow(ctx, p) - auth, err := fs.getUserAuth(ctx, u, "") + // in order to remove the folder or the file without + // moving it to the recycle bin, we should take + // the privileges of the root + auth, err := fs.getRootAuth(ctx) if err != nil { return err } - return fs.c.Remove(ctx, auth, fn) + return fs.c.Remove(ctx, auth, fn, true) } return errors.New("eosfs: shadow delete of share folder that is neither root nor child. path=" + p) From 147c0c2c1c000b636c0a425afbec68d029975045 Mon Sep 17 00:00:00 2001 From: David Christofas Date: Mon, 6 Dec 2021 11:09:02 +0100 Subject: [PATCH 03/58] Public link propfind (#2315) --- changelog/unreleased/public-link-propfind.md | 6 ++++++ .../http/services/owncloud/ocdav/propfind.go | 18 ++++++++++++++++++ .../http/services/owncloud/ocdav/publicfile.go | 3 --- 3 files changed, 24 insertions(+), 3 deletions(-) create mode 100644 changelog/unreleased/public-link-propfind.md diff --git a/changelog/unreleased/public-link-propfind.md b/changelog/unreleased/public-link-propfind.md new file mode 100644 index 0000000000..d2809bbdb0 --- /dev/null +++ b/changelog/unreleased/public-link-propfind.md @@ -0,0 +1,6 @@ +Enhancement: Add new attributes to public link propfinds + +Added a new property "oc:signature-auth" to public link propfinds. +This is a necessary change to be able to support archive downloads in password protected public links. + +https://github.com/cs3org/reva/pull/2315 diff --git a/internal/http/services/owncloud/ocdav/propfind.go b/internal/http/services/owncloud/ocdav/propfind.go index e1c43146b6..320049dfe1 100644 --- a/internal/http/services/owncloud/ocdav/propfind.go +++ b/internal/http/services/owncloud/ocdav/propfind.go @@ -847,6 +847,24 @@ func (s *svc) mdToPropResponse(ctx context.Context, pf *propfindXML, md *provide } else { propstatNotFound.Prop = append(propstatNotFound.Prop, s.newProp("oc:"+pf.Prop[i].Local, "")) } + case "signature-auth": + if isPublic { + // We only want to add the attribute to the root of the propfind. + if strings.HasSuffix(md.Path, ls.Token) && ls.Signature != nil { + expiration := time.Unix(int64(ls.Signature.SignatureExpiration.Seconds), int64(ls.Signature.SignatureExpiration.Nanos)) + var sb strings.Builder + sb.WriteString("") + sb.WriteString(ls.Signature.Signature) + sb.WriteString("") + sb.WriteString("") + sb.WriteString(expiration.Format(time.RFC3339)) + sb.WriteString("") + + propstatOK.Prop = append(propstatOK.Prop, s.newPropRaw("oc:signature-auth", sb.String())) + } else { + propstatNotFound.Prop = append(propstatNotFound.Prop, s.newProp("oc:signature-auth", "")) + } + } case "privatelink": // phoenix only // https://phoenix.owncloud.com/f/9 fallthrough diff --git a/internal/http/services/owncloud/ocdav/publicfile.go b/internal/http/services/owncloud/ocdav/publicfile.go index b369f6650b..ed75e31672 100644 --- a/internal/http/services/owncloud/ocdav/publicfile.go +++ b/internal/http/services/owncloud/ocdav/publicfile.go @@ -181,9 +181,6 @@ func (s *svc) handlePropfindOnToken(w http.ResponseWriter, r *http.Request, ns s w.WriteHeader(http.StatusNotFound) return } - // adjust path - tokenStatInfo.Path = path.Join("/", tokenStatInfo.Path, path.Base(pathRes.Path)) - infos := s.getPublicFileInfos(onContainer, depth == "0", tokenStatInfo) propRes, err := s.multistatusResponse(ctx, &pf, infos, ns, nil) From 7f7f9ec8cc33d536e89e72fb7905d88921455dce Mon Sep 17 00:00:00 2001 From: David Christofas Date: Tue, 7 Dec 2021 12:28:19 +0100 Subject: [PATCH 04/58] fix public share type in propfinds (#2316) --- changelog/unreleased/fix-propfind-sharetype.md | 5 +++++ internal/http/services/owncloud/ocdav/propfind.go | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 changelog/unreleased/fix-propfind-sharetype.md diff --git a/changelog/unreleased/fix-propfind-sharetype.md b/changelog/unreleased/fix-propfind-sharetype.md new file mode 100644 index 0000000000..35f6aa4dd7 --- /dev/null +++ b/changelog/unreleased/fix-propfind-sharetype.md @@ -0,0 +1,5 @@ +Bugfix: Fix the share types in propfinds + +The share types for public links were not correctly added to propfinds. + +https://github.com/cs3org/reva/pull/2316 diff --git a/internal/http/services/owncloud/ocdav/propfind.go b/internal/http/services/owncloud/ocdav/propfind.go index 320049dfe1..eb0e0b197f 100644 --- a/internal/http/services/owncloud/ocdav/propfind.go +++ b/internal/http/services/owncloud/ocdav/propfind.go @@ -159,7 +159,7 @@ func (s *svc) propfindResponse(ctx context.Context, w http.ResponseWriter, r *ht var linkshares map[string]struct{} listResp, err := client.ListPublicShares(ctx, &link.ListPublicSharesRequest{Filters: filters}) if err == nil { - linkshares := make(map[string]struct{}) + linkshares = make(map[string]struct{}, len(listResp.Share)) for i := range listResp.Share { linkshares[listResp.Share[i].ResourceId.OpaqueId] = struct{}{} } From d9eefe2937896b7c93a9c43813af12fa504bcb10 Mon Sep 17 00:00:00 2001 From: Swikriti Tripathi <41103328+SwikritiT@users.noreply.github.com> Date: Wed, 8 Dec 2021 14:39:34 +0545 Subject: [PATCH 05/58] Bump core commit id for tests (#2331) --- .drone.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.env b/.drone.env index 549174112e..cce5230458 100644 --- a/.drone.env +++ b/.drone.env @@ -1,3 +1,3 @@ # The test runner source for API tests -CORE_COMMITID=918845e387e18e0ecd765458bc721b2b571b9024 +CORE_COMMITID=4bbd91de68aefdf94c03d2920a3353702f856280 CORE_BRANCH=acceptance-test-changes-waiting-2021-11 From 4dde766d9e015ee4c875e0e8312003fa054d248e Mon Sep 17 00:00:00 2001 From: Ishank Arora Date: Thu, 9 Dec 2021 14:04:24 +0530 Subject: [PATCH 06/58] Revert "Fix content disposition (#2303)" (#2332) This reverts commit 3cba22371b78213f2e49197c2783220331a264bd. --- changelog/unreleased/fix-content-disposition.md | 3 ++- .../publicstorageprovider/publicstorageprovider.go | 7 ++++++- internal/http/services/owncloud/ocdav/publicfile.go | 4 ---- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/changelog/unreleased/fix-content-disposition.md b/changelog/unreleased/fix-content-disposition.md index dcd887266d..58f126a7ac 100644 --- a/changelog/unreleased/fix-content-disposition.md +++ b/changelog/unreleased/fix-content-disposition.md @@ -1,4 +1,5 @@ Bugfix: Fix content disposition header for public links files https://github.com/cs3org/reva/pull/2303 -https://github.com/cs3org/reva/pull/2297 \ No newline at end of file +https://github.com/cs3org/reva/pull/2297 +https://github.com/cs3org/reva/pull/2332 \ No newline at end of file diff --git a/internal/grpc/services/publicstorageprovider/publicstorageprovider.go b/internal/grpc/services/publicstorageprovider/publicstorageprovider.go index 4a8185dc47..c6ec4e983e 100644 --- a/internal/grpc/services/publicstorageprovider/publicstorageprovider.go +++ b/internal/grpc/services/publicstorageprovider/publicstorageprovider.go @@ -163,7 +163,12 @@ func (s *service) translatePublicRefToCS3Ref(ctx context.Context, ref *provider. return nil, "", nil, st, nil } - cs3Ref := &provider.Reference{Path: path.Join("/", shareInfo.Path, relativePath)} + p := shareInfo.Path + if shareInfo.Type != provider.ResourceType_RESOURCE_TYPE_FILE { + p = path.Join("/", shareInfo.Path, relativePath) + } + cs3Ref := &provider.Reference{Path: p} + log.Debug(). Interface("sourceRef", ref). Interface("cs3Ref", cs3Ref). diff --git a/internal/http/services/owncloud/ocdav/publicfile.go b/internal/http/services/owncloud/ocdav/publicfile.go index ed75e31672..b7a62d8bc7 100644 --- a/internal/http/services/owncloud/ocdav/publicfile.go +++ b/internal/http/services/owncloud/ocdav/publicfile.go @@ -55,7 +55,6 @@ func (h *PublicFileHandler) Handler(s *svc) http.Handler { return } - r.URL.Path = path.Base(r.URL.Path) switch r.Method { case MethodPropfind: s.handlePropfindOnToken(w, r, h.namespace, false) @@ -121,9 +120,6 @@ func (s *svc) adjustResourcePathInURL(w http.ResponseWriter, r *http.Request) bo return false } - // adjust path in request URL to point at the parent - r.URL.Path = path.Dir(r.URL.Path) - return true } From 8fde5db7c6484d7c8261db6019da80ad1172c674 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 9 Dec 2021 14:43:38 +0530 Subject: [PATCH 07/58] [Build-deps]: Bump github.com/gomodule/redigo from 1.8.5 to 1.8.6 (#2326) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 1e6cbb7387..a76d7ed305 100644 --- a/go.mod +++ b/go.mod @@ -27,7 +27,7 @@ require ( github.com/go-sql-driver/mysql v1.6.0 github.com/golang-jwt/jwt v3.2.2+incompatible github.com/golang/protobuf v1.5.2 - github.com/gomodule/redigo v1.8.5 + github.com/gomodule/redigo v1.8.6 github.com/google/go-cmp v0.5.6 github.com/google/go-github v17.0.0+incompatible github.com/google/go-querystring v1.1.0 // indirect diff --git a/go.sum b/go.sum index 7b3613ae08..3061cb2797 100644 --- a/go.sum +++ b/go.sum @@ -313,8 +313,8 @@ github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/gomodule/redigo v1.8.5 h1:nRAxCa+SVsyjSBrtZmG/cqb6VbTmuRzpg/PoTFlpumc= -github.com/gomodule/redigo v1.8.5/go.mod h1:P9dn9mFrCBvWhGE1wpxx6fgq7BAeLBk+UUUzlpkBYO0= +github.com/gomodule/redigo v1.8.6 h1:h7kHSqUl2kxeaQtVslsfUCPJ1oz2pxcyzLy4zezIzPw= +github.com/gomodule/redigo v1.8.6/go.mod h1:P9dn9mFrCBvWhGE1wpxx6fgq7BAeLBk+UUUzlpkBYO0= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= From e2109124d15549d54a40d3c39e63cde1dfcc9223 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 9 Dec 2021 14:43:51 +0530 Subject: [PATCH 08/58] [Build-deps]: Bump github.com/mitchellh/mapstructure from 1.4.2 to 1.4.3 (#2324) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index a76d7ed305..e2428123a2 100644 --- a/go.mod +++ b/go.mod @@ -44,7 +44,7 @@ require ( github.com/minio/minio-go/v7 v7.0.16 github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/go-testing-interface v1.14.1 // indirect - github.com/mitchellh/mapstructure v1.4.2 + github.com/mitchellh/mapstructure v1.4.3 github.com/onsi/ginkgo v1.16.5 github.com/onsi/gomega v1.17.0 github.com/pkg/errors v0.9.1 diff --git a/go.sum b/go.sum index 3061cb2797..d7042900ca 100644 --- a/go.sum +++ b/go.sum @@ -487,8 +487,8 @@ github.com/mitchellh/mapstructure v1.3.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RR github.com/mitchellh/mapstructure v1.3.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.4.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/mitchellh/mapstructure v1.4.2 h1:6h7AQ0yhTcIsmFmnAwQls75jp2Gzs4iB8W7pjMO+rqo= -github.com/mitchellh/mapstructure v1.4.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/mapstructure v1.4.3 h1:OVowDSCllw/YjdLkam3/sm7wEtOy59d8ndGgCcyj8cs= +github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= From 66071c8e76a3f6aa9326e459025cd14bf80ac7cb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 9 Dec 2021 14:44:02 +0530 Subject: [PATCH 09/58] [Build-deps]: Bump github.com/aws/aws-sdk-go from 1.42.9 to 1.42.19 (#2325) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index e2428123a2..167002151e 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/Masterminds/sprig v2.22.0+incompatible github.com/ReneKroon/ttlcache/v2 v2.9.0 github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect - github.com/aws/aws-sdk-go v1.42.9 + github.com/aws/aws-sdk-go v1.42.19 github.com/beevik/etree v1.1.0 github.com/bluele/gcache v0.0.2 github.com/c-bata/go-prompt v0.2.5 diff --git a/go.sum b/go.sum index d7042900ca..dd5a0265b6 100644 --- a/go.sum +++ b/go.sum @@ -77,8 +77,8 @@ github.com/aws/aws-sdk-go v1.20.1/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN github.com/aws/aws-sdk-go v1.34.28/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48= github.com/aws/aws-sdk-go v1.38.35/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/aws/aws-sdk-go v1.40.11/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q= -github.com/aws/aws-sdk-go v1.42.9 h1:8ptAGgA+uC2TUbdvUeOVSfBocIZvGE2NKiLxkAcn1GA= -github.com/aws/aws-sdk-go v1.42.9/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q= +github.com/aws/aws-sdk-go v1.42.19 h1:L/aM1QwsqVia9qIqexTHwYN+lgLYuOtf11VDgz0YIyw= +github.com/aws/aws-sdk-go v1.42.19/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q= github.com/beevik/etree v1.1.0 h1:T0xke/WvNtMoCqgzPhkX2r4rjY3GDZFi+FjpRZY2Jbs= github.com/beevik/etree v1.1.0/go.mod h1:r8Aw8JqVegEf0w2fDnATrX9VpkMcyFeM0FhwO62wh+A= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= From 68b64bf2be8c703f7d5e12d8979cba78077e0711 Mon Sep 17 00:00:00 2001 From: Willy Kloucek <34452982+wkloucek@users.noreply.github.com> Date: Thu, 9 Dec 2021 14:48:20 +0100 Subject: [PATCH 10/58] fix app provider new file action and improve app provider error codes (#2210) --- changelog/unreleased/app-provider-new-file.md | 8 + internal/grpc/services/gateway/appprovider.go | 7 +- .../http/services/appprovider/appprovider.go | 260 +++++++++++------- internal/http/services/appprovider/errors.go | 78 ++++++ internal/http/services/archiver/handler.go | 32 +-- internal/http/services/owncloud/ocdav/get.go | 3 +- internal/http/services/owncloud/ocdav/head.go | 3 +- internal/http/services/owncloud/ocdav/meta.go | 3 +- internal/http/services/owncloud/ocdav/move.go | 3 +- .../http/services/owncloud/ocdav/ocdav.go | 38 --- .../services/owncloud/ocdav/ocdav_test.go | 58 +--- .../http/services/owncloud/ocdav/propfind.go | 7 +- internal/http/services/owncloud/ocdav/put.go | 3 +- .../http/services/owncloud/ocdav/trashbin.go | 3 +- internal/http/services/owncloud/ocdav/tus.go | 3 +- .../http/services/owncloud/ocdav/versions.go | 3 +- .../handlers/apps/sharing/shares/shares.go | 20 +- pkg/utils/resourceid/owncloud.go | 77 ++++++ pkg/utils/resourceid/owncloud_test.go | 88 ++++++ 19 files changed, 449 insertions(+), 248 deletions(-) create mode 100644 changelog/unreleased/app-provider-new-file.md create mode 100644 internal/http/services/appprovider/errors.go create mode 100644 pkg/utils/resourceid/owncloud.go create mode 100644 pkg/utils/resourceid/owncloud_test.go diff --git a/changelog/unreleased/app-provider-new-file.md b/changelog/unreleased/app-provider-new-file.md new file mode 100644 index 0000000000..223070bfac --- /dev/null +++ b/changelog/unreleased/app-provider-new-file.md @@ -0,0 +1,8 @@ +Change: Fix app provider new file creation and improved error codes + +We've fixed the behavior for the app provider when creating new files. +Previously the app provider would overwrite already existing files when creating a new file, this is now handled and prevented. +The new file endpoint accepted a path to a file, but this does not work for spaces. Therefore we now use the resource id of the folder where the file should be created and a filename to create the new file. +Also the app provider returns more useful error codes in a lot of cases. + +https://github.com/cs3org/reva/pull/2210 diff --git a/internal/grpc/services/gateway/appprovider.go b/internal/grpc/services/gateway/appprovider.go index 74e8a5adec..cbcad414c2 100644 --- a/internal/grpc/services/gateway/appprovider.go +++ b/internal/grpc/services/gateway/appprovider.go @@ -242,7 +242,7 @@ func (s *svc) findAppProvider(ctx context.Context, ri *storageprovider.ResourceI // we did not find a default provider if res.Status.Code == rpc.Code_CODE_NOT_FOUND { - err := errtypes.NotFound(fmt.Sprintf("gateway: default app rovider for mime type:%s not found", ri.MimeType)) + err := errtypes.NotFound(fmt.Sprintf("gateway: default app provider for mime type:%s not found", ri.MimeType)) return nil, err } @@ -285,7 +285,10 @@ func (s *svc) findAppProvider(ctx context.Context, ri *storageprovider.ResourceI } res.Providers = filteredProviders - // if we only have one app provider we verify that it matches the requested app name + if len(res.Providers) == 0 { + return nil, errtypes.NotFound(fmt.Sprintf("app '%s' not found", app)) + } + if len(res.Providers) == 1 { return res.Providers[0], nil } diff --git a/internal/http/services/appprovider/appprovider.go b/internal/http/services/appprovider/appprovider.go index 9795b1302e..ed101a0f3c 100644 --- a/internal/http/services/appprovider/appprovider.go +++ b/internal/http/services/appprovider/appprovider.go @@ -19,13 +19,9 @@ package appprovider import ( - "context" - "encoding/base64" "encoding/json" - "fmt" "net/http" - "strings" - "unicode/utf8" + "path" appregistry "github.com/cs3org/go-cs3apis/cs3/app/registry/v1beta1" gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1" @@ -33,8 +29,6 @@ import ( provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" typespb "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" "github.com/cs3org/reva/internal/http/services/datagateway" - "github.com/cs3org/reva/internal/http/services/ocmd" - "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/rgrpc/status" "github.com/cs3org/reva/pkg/rgrpc/todo/pool" "github.com/cs3org/reva/pkg/rhttp" @@ -42,15 +36,11 @@ import ( "github.com/cs3org/reva/pkg/rhttp/router" "github.com/cs3org/reva/pkg/sharedconf" "github.com/cs3org/reva/pkg/utils" + "github.com/cs3org/reva/pkg/utils/resourceid" ua "github.com/mileusna/useragent" "github.com/mitchellh/mapstructure" "github.com/pkg/errors" "github.com/rs/zerolog" - "github.com/rs/zerolog/log" -) - -const ( - idDelimiter string = ":" ) func init() { @@ -61,6 +51,7 @@ func init() { type Config struct { Prefix string `mapstructure:"prefix"` GatewaySvc string `mapstructure:"gatewaysvc"` + Insecure bool `mapstructure:"insecure"` } func (c *Config) init() { @@ -115,17 +106,17 @@ func (s *svc) Handler() http.Handler { case "open": s.handleOpen(w, r) default: - ocmd.WriteError(w, r, ocmd.APIErrorServerError, "unsupported POST endpoint", nil) + writeError(w, r, appErrorUnimplemented, "unsupported POST endpoint", nil) } case "GET": switch head { case "list": s.handleList(w, r) default: - ocmd.WriteError(w, r, ocmd.APIErrorServerError, "unsupported GET endpoint", nil) + writeError(w, r, appErrorUnimplemented, "unsupported GET endpoint", nil) } default: - ocmd.WriteError(w, r, ocmd.APIErrorServerError, "unsupported method", nil) + writeError(w, r, appErrorUnimplemented, "unsupported method", nil) } }) } @@ -135,34 +126,86 @@ func (s *svc) handleNew(w http.ResponseWriter, r *http.Request) { client, err := pool.GetGatewayServiceClient(s.conf.GatewaySvc) if err != nil { - ocmd.WriteError(w, r, ocmd.APIErrorServerError, "error getting grpc gateway client", err) + writeError(w, r, appErrorServerError, "error getting grpc gateway client", err) return } if r.URL.Query().Get("template") != "" { // TODO in the future we want to create a file out of the given template - ocmd.WriteError(w, r, ocmd.APIErrorInvalidParameter, "Template not implemented", - errtypes.NotSupported("Templates are not yet supported")) + writeError(w, r, appErrorUnimplemented, "template is not implemented", nil) + return + } + + parentContainerID := r.URL.Query().Get("parent_container_id") + if parentContainerID == "" { + writeError(w, r, appErrorInvalidParameter, "missing parent container ID", nil) + return + } + + parentContainerRef := resourceid.OwnCloudResourceIDUnwrap(parentContainerID) + if parentContainerRef == nil { + writeError(w, r, appErrorInvalidParameter, "invalid parent container ID", nil) + return + } + + filename := r.URL.Query().Get("filename") + if filename == "" { + writeError(w, r, appErrorInvalidParameter, "missing filename", nil) + return + } + + dirPart, filePart := path.Split(filename) + if dirPart != "" || filePart != filename { + writeError(w, r, appErrorInvalidParameter, "the filename must not contain a path segment", nil) + return + } + + statParentContainerReq := &provider.StatRequest{ + Ref: &provider.Reference{ + ResourceId: parentContainerRef, + }, + } + parentContainer, err := client.Stat(ctx, statParentContainerReq) + if err != nil { + writeError(w, r, appErrorServerError, "error sending a grpc stat request", err) + return + } + + if parentContainer.Status.Code != rpc.Code_CODE_OK { + writeError(w, r, appErrorNotFound, "the parent container is not accessible or does not exist", err) return } - target := r.URL.Query().Get("filename") - if target == "" { - ocmd.WriteError(w, r, ocmd.APIErrorInvalidParameter, "Missing filename", - errtypes.UserRequired("Missing filename")) + if parentContainer.Info.Type != provider.ResourceType_RESOURCE_TYPE_CONTAINER { + writeError(w, r, appErrorInvalidParameter, "the parent container id does not point to a container", nil) return } - // TODO(lopresti) if target is relative, currently the gateway fails to identify a storage provider (?) - // and just returns a CODE_INTERNAL error on InitiateFileUpload. - // Therefore for now make sure the target is absolute. - if target[0] != '/' { - target = "/" + target + fileRef := &provider.Reference{ + Path: path.Join(parentContainer.Info.Path, utils.MakeRelativePath(filename)), + } + + statFileReq := &provider.StatRequest{ + Ref: fileRef, + } + statFileRes, err := client.Stat(ctx, statFileReq) + if err != nil { + writeError(w, r, appErrorServerError, "failed to stat the file", err) + return + } + + if statFileRes.Status.Code != rpc.Code_CODE_NOT_FOUND { + if statFileRes.Status.Code == rpc.Code_CODE_OK { + writeError(w, r, appErrorAlreadyExists, "the file already exists", nil) + return + } + writeError(w, r, appErrorServerError, "statting the file returned unexpected status code", err) + return } // Create empty file via storageprovider createReq := &provider.InitiateFileUploadRequest{ - Ref: &provider.Reference{Path: target}, + Ref: fileRef, Opaque: &typespb.Opaque{ Map: map[string]*typespb.OpaqueEntry{ "Upload-Length": { @@ -172,13 +215,16 @@ func (s *svc) handleNew(w http.ResponseWriter, r *http.Request) { }, }, } + + // having a client.CreateFile() function would come in handy here... + createRes, err := client.InitiateFileUpload(ctx, createReq) if err != nil { - ocmd.WriteError(w, r, ocmd.APIErrorServerError, "error calling InitiateFileUpload", err) + writeError(w, r, appErrorServerError, "error calling InitiateFileUpload", err) return } if createRes.Status.Code != rpc.Code_CODE_OK { - ocmd.WriteError(w, r, ocmd.APIErrorServerError, "error creating resource", status.NewErrorFromCode(createRes.Status.Code, "appprovider")) + writeError(w, r, appErrorServerError, "error calling InitiateFileUpload", nil) return } @@ -191,44 +237,58 @@ func (s *svc) handleNew(w http.ResponseWriter, r *http.Request) { } httpReq, err := rhttp.NewRequest(ctx, http.MethodPut, ep, nil) if err != nil { - ocmd.WriteError(w, r, ocmd.APIErrorServerError, "error executing PUT", err) + writeError(w, r, appErrorServerError, "failed to create the file", err) return } httpReq.Header.Set(datagateway.TokenTransportHeader, token) - httpRes, err := rhttp.GetHTTPClient().Do(httpReq) + httpRes, err := rhttp.GetHTTPClient( + rhttp.Context(ctx), + rhttp.Insecure(s.conf.Insecure), + ).Do(httpReq) if err != nil { - log.Error().Err(err).Msg("error doing PUT request to data service") - ocmd.WriteError(w, r, ocmd.APIErrorServerError, "error executing PUT", err) + writeError(w, r, appErrorServerError, "failed to create the file", err) return } defer httpRes.Body.Close() - if httpRes.StatusCode != http.StatusOK { - log.Error().Msg("PUT request to data server failed") - ocmd.WriteError(w, r, ocmd.APIErrorServerError, "error executing PUT", - errtypes.InternalError(fmt.Sprint(httpRes.StatusCode))) + if httpRes.StatusCode == http.StatusForbidden { + // the file upload was already finished since it is a zero byte file + // TODO: why do we get a 401 then!? + } else if httpRes.StatusCode != http.StatusOK { + writeError(w, r, appErrorServerError, "failed to create the file", nil) return } // Stat the newly created file - statRes, ocmderr, err := statRef(ctx, provider.Reference{Path: target}, client) + statRes, err := client.Stat(ctx, statFileReq) if err != nil { - log.Error().Err(err).Msg("error statting created file") - ocmd.WriteError(w, r, ocmderr, "Created file not found", errtypes.NotFound("Created file not found")) + writeError(w, r, appErrorServerError, "statting the created file failed", err) + return + } + + if statRes.Status.Code != rpc.Code_CODE_OK { + writeError(w, r, appErrorServerError, "statting the created file failed", nil) + return + } + + if statRes.Info.Type != provider.ResourceType_RESOURCE_TYPE_FILE { + writeError(w, r, appErrorInvalidParameter, "the given file id does not point to a file", nil) return } - // Base64-encode the fileid for the web to consume it - b64id := base64.StdEncoding.EncodeToString([]byte(statRes.Id.StorageId + idDelimiter + statRes.Id.OpaqueId)) - js, err := json.Marshal(map[string]interface{}{"file_id": b64id}) + js, err := json.Marshal( + map[string]interface{}{ + "file_id": resourceid.OwnCloudResourceIDWrap(statRes.Info.Id), + }, + ) if err != nil { - ocmd.WriteError(w, r, ocmd.APIErrorServerError, "error marshalling JSON response", err) + writeError(w, r, appErrorServerError, "error marshalling JSON response", err) return } w.Header().Set("Content-Type", "application/json") if _, err = w.Write(js); err != nil { - ocmd.WriteError(w, r, ocmd.APIErrorServerError, "error writing JSON response", err) + writeError(w, r, appErrorServerError, "error writing JSON response", err) return } } @@ -237,31 +297,30 @@ func (s *svc) handleList(w http.ResponseWriter, r *http.Request) { ctx := r.Context() client, err := pool.GetGatewayServiceClient(s.conf.GatewaySvc) if err != nil { - ocmd.WriteError(w, r, ocmd.APIErrorServerError, "error getting grpc gateway client", err) + writeError(w, r, appErrorServerError, "error getting grpc gateway client", err) return } listRes, err := client.ListSupportedMimeTypes(ctx, &appregistry.ListSupportedMimeTypesRequest{}) if err != nil { - ocmd.WriteError(w, r, ocmd.APIErrorServerError, "error listing supported mime types", err) + writeError(w, r, appErrorServerError, "error listing supported mime types", err) return } if listRes.Status.Code != rpc.Code_CODE_OK { - ocmd.WriteError(w, r, ocmd.APIErrorServerError, "error listing supported mime types", - status.NewErrorFromCode(listRes.Status.Code, "appprovider")) + writeError(w, r, appErrorServerError, "error listing supported mime types", nil) return } res := filterAppsByUserAgent(listRes.MimeTypes, r.UserAgent()) js, err := json.Marshal(map[string]interface{}{"mime-types": res}) if err != nil { - ocmd.WriteError(w, r, ocmd.APIErrorServerError, "error marshalling JSON response", err) + writeError(w, r, appErrorServerError, "error marshalling JSON response", err) return } w.Header().Set("Content-Type", "application/json") if _, err = w.Write(js); err != nil { - ocmd.WriteError(w, r, ocmd.APIErrorServerError, "error writing JSON response", err) + writeError(w, r, appErrorServerError, "error writing JSON response", err) return } } @@ -271,43 +330,83 @@ func (s *svc) handleOpen(w http.ResponseWriter, r *http.Request) { client, err := pool.GetGatewayServiceClient(s.conf.GatewaySvc) if err != nil { - ocmd.WriteError(w, r, ocmd.APIErrorServerError, "Internal error with the gateway, please try again later", err) + writeError(w, r, appErrorServerError, "Internal error with the gateway, please try again later", err) + return + } + + fileID := r.URL.Query().Get("file_id") + + if fileID == "" { + writeError(w, r, appErrorInvalidParameter, "missing file ID", nil) return } - info, errCode, err := s.getStatInfo(ctx, r.URL.Query().Get("file_id"), client) + resourceID := resourceid.OwnCloudResourceIDUnwrap(fileID) + if resourceID == nil { + writeError(w, r, appErrorInvalidParameter, "invalid file ID", nil) + return + } + + fileRef := &provider.Reference{ + ResourceId: resourceID, + } + + statRes, err := client.Stat(ctx, &provider.StatRequest{Ref: fileRef}) if err != nil { - ocmd.WriteError(w, r, errCode, "Internal error accessing the file, please try again later", err) + writeError(w, r, appErrorServerError, "Internal error accessing the file, please try again later", err) + return + } + + if statRes.Status.Code == rpc.Code_CODE_NOT_FOUND { + writeError(w, r, appErrorNotFound, "file does not exist", nil) + return + } else if statRes.Status.Code != rpc.Code_CODE_OK { + writeError(w, r, appErrorServerError, "failed to stat the file", nil) + return + } + + if statRes.Info.Type != provider.ResourceType_RESOURCE_TYPE_FILE { + writeError(w, r, appErrorInvalidParameter, "the given file id does not point to a file", nil) + return + } + + viewMode := getViewMode(statRes.Info, r.URL.Query().Get("view_mode")) + if viewMode == gateway.OpenInAppRequest_VIEW_MODE_INVALID { + writeError(w, r, appErrorInvalidParameter, "invalid view mode", err) return } openReq := gateway.OpenInAppRequest{ - Ref: &provider.Reference{ResourceId: info.Id}, - ViewMode: getViewMode(info, r.URL.Query().Get("view_mode")), + Ref: fileRef, + ViewMode: viewMode, App: r.URL.Query().Get("app_name"), } openRes, err := client.OpenInApp(ctx, &openReq) if err != nil { - ocmd.WriteError(w, r, ocmd.APIErrorServerError, + writeError(w, r, appErrorServerError, "Error contacting the requested application, please use a different one or try again later", err) return } if openRes.Status.Code != rpc.Code_CODE_OK { - ocmd.WriteError(w, r, ocmd.APIErrorServerError, openRes.Status.Message, - status.NewErrorFromCode(openRes.Status.Code, "Error calling OpenInApp")) + if openRes.Status.Code == rpc.Code_CODE_NOT_FOUND { + writeError(w, r, appErrorNotFound, openRes.Status.Message, nil) + return + } + writeError(w, r, appErrorServerError, openRes.Status.Message, + status.NewErrorFromCode(openRes.Status.Code, "error calling OpenInApp")) return } js, err := json.Marshal(openRes.AppUrl) if err != nil { - ocmd.WriteError(w, r, ocmd.APIErrorServerError, "Internal error with JSON payload", + writeError(w, r, appErrorServerError, "Internal error with JSON payload", errors.Wrap(err, "error marshalling JSON response")) return } w.Header().Set("Content-Type", "application/json") if _, err = w.Write(js); err != nil { - ocmd.WriteError(w, r, ocmd.APIErrorServerError, "Internal error with JSON payload", + writeError(w, r, appErrorServerError, "Internal error with JSON payload", errors.Wrap(err, "error writing JSON response")) return } @@ -334,43 +433,6 @@ func filterAppsByUserAgent(mimeTypes []*appregistry.MimeTypeInfo, userAgent stri return res } -func (s *svc) getStatInfo(ctx context.Context, fileID string, client gateway.GatewayAPIClient) (*provider.ResourceInfo, ocmd.APIErrorCode, error) { - if fileID == "" { - return nil, ocmd.APIErrorInvalidParameter, errors.New("fileID parameter missing in request") - } - - decodedID, err := base64.URLEncoding.DecodeString(fileID) - if err != nil { - return nil, ocmd.APIErrorInvalidParameter, errors.Wrap(err, fmt.Sprintf("fileID %s doesn't follow the required format", fileID)) - } - - parts := strings.Split(string(decodedID), idDelimiter) - if !utf8.ValidString(parts[0]) || !utf8.ValidString(parts[1]) { - return nil, ocmd.APIErrorInvalidParameter, errtypes.BadRequest(fmt.Sprintf("fileID %s contains illegal characters", fileID)) - } - res := &provider.ResourceId{ - StorageId: parts[0], - OpaqueId: parts[1], - } - - return statRef(ctx, provider.Reference{ResourceId: res}, client) -} - -func statRef(ctx context.Context, ref provider.Reference, client gateway.GatewayAPIClient) (*provider.ResourceInfo, ocmd.APIErrorCode, error) { - statReq := provider.StatRequest{Ref: &ref} - statRes, err := client.Stat(ctx, &statReq) - if err != nil { - return nil, ocmd.APIErrorServerError, err - } - if statRes.Status.Code != rpc.Code_CODE_OK { - return nil, ocmd.APIErrorServerError, status.NewErrorFromCode(statRes.Status.Code, "appprovider") - } - if statRes.Info.Type != provider.ResourceType_RESOURCE_TYPE_FILE { - return nil, ocmd.APIErrorServerError, errors.New("unsupported resource type") - } - return statRes.Info, ocmd.APIErrorCode(""), nil -} - func getViewMode(res *provider.ResourceInfo, vm string) gateway.OpenInAppRequest_ViewMode { if vm != "" { return utils.GetViewMode(vm) diff --git a/internal/http/services/appprovider/errors.go b/internal/http/services/appprovider/errors.go new file mode 100644 index 0000000000..353925423d --- /dev/null +++ b/internal/http/services/appprovider/errors.go @@ -0,0 +1,78 @@ +// Copyright 2018-2021 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package appprovider + +import ( + "encoding/json" + "net/http" + + "github.com/cs3org/reva/pkg/appctx" +) + +// appErrorCode stores the type of error encountered +type appErrorCode string + +const ( + appErrorNotFound appErrorCode = "RESOURCE_NOT_FOUND" + appErrorAlreadyExists appErrorCode = "RESOURCE_ALREADY_EXISTS" + appErrorUnauthenticated appErrorCode = "UNAUTHENTICATED" + appErrorUnimplemented appErrorCode = "NOT_IMPLEMENTED" + appErrorInvalidParameter appErrorCode = "INVALID_PARAMETER" + appErrorServerError appErrorCode = "SERVER_ERROR" +) + +// appErrorCodeMapping stores the HTTP error code mapping for various APIErrorCodes +var appErrorCodeMapping = map[appErrorCode]int{ + appErrorNotFound: http.StatusNotFound, + appErrorAlreadyExists: http.StatusForbidden, + appErrorUnauthenticated: http.StatusUnauthorized, + appErrorUnimplemented: http.StatusNotImplemented, + appErrorInvalidParameter: http.StatusBadRequest, + appErrorServerError: http.StatusInternalServerError, +} + +// APIError encompasses the error type and message +type appError struct { + Code appErrorCode `json:"code"` + Message string `json:"message"` +} + +// writeError handles writing error responses +func writeError(w http.ResponseWriter, r *http.Request, code appErrorCode, message string, err error) { + if err != nil { + appctx.GetLogger(r.Context()).Error().Err(err).Msg(message) + } + + var encoded []byte + w.Header().Set("Content-Type", "application/json") + encoded, err = json.MarshalIndent(appError{Code: code, Message: message}, "", " ") + + if err != nil { + appctx.GetLogger(r.Context()).Error().Err(err).Msg("error encoding response") + w.WriteHeader(http.StatusInternalServerError) + return + } + + w.WriteHeader(appErrorCodeMapping[code]) + _, err = w.Write(encoded) + if err != nil { + appctx.GetLogger(r.Context()).Error().Err(err).Msg("error writing response") + w.WriteHeader(http.StatusInternalServerError) + } +} diff --git a/internal/http/services/archiver/handler.go b/internal/http/services/archiver/handler.go index ffa8df0356..ee2c05f111 100644 --- a/internal/http/services/archiver/handler.go +++ b/internal/http/services/archiver/handler.go @@ -20,12 +20,10 @@ package archiver import ( "context" - "encoding/base64" + "errors" "fmt" "net/http" - "strings" "time" - "unicode/utf8" gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1" rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" @@ -41,6 +39,7 @@ import ( "github.com/cs3org/reva/pkg/sharedconf" "github.com/cs3org/reva/pkg/storage/utils/downloader" "github.com/cs3org/reva/pkg/storage/utils/walker" + "github.com/cs3org/reva/pkg/utils/resourceid" "github.com/gdexlab/go-render/render" ua "github.com/mileusna/useragent" "github.com/mitchellh/mapstructure" @@ -130,17 +129,14 @@ func (s *svc) getFiles(ctx context.Context, files, ids []string) ([]string, erro for _, id := range ids { // id is base64 encoded and after decoding has the form : - storageID, opaqueID, err := decodeResourceID(id) - if err != nil { - return nil, err + ref := resourceid.OwnCloudResourceIDUnwrap(id) + if ref == nil { + return nil, errors.New("could not unwrap given file id") } resp, err := s.gtwClient.Stat(ctx, &provider.StatRequest{ Ref: &provider.Reference{ - ResourceId: &provider.ResourceId{ - StorageId: storageID, - OpaqueId: opaqueID, - }, + ResourceId: ref, }, }) @@ -279,19 +275,3 @@ func (s *svc) Close() error { func (s *svc) Unprotected() []string { return nil } - -func decodeResourceID(encodedID string) (string, string, error) { - decodedID, err := base64.URLEncoding.DecodeString(encodedID) - if err != nil { - return "", "", errtypes.BadRequest("resource ID does not follow the required format") - } - - parts := strings.Split(string(decodedID), ":") - if len(parts) != 2 { - return "", "", errtypes.BadRequest("resource ID does not follow the required format") - } - if !utf8.ValidString(parts[0]) || !utf8.ValidString(parts[1]) { - return "", "", errtypes.BadRequest("resourceID contains illegal characters") - } - return parts[0], parts[1], nil -} diff --git a/internal/http/services/owncloud/ocdav/get.go b/internal/http/services/owncloud/ocdav/get.go index cefc4da7df..a812365846 100644 --- a/internal/http/services/owncloud/ocdav/get.go +++ b/internal/http/services/owncloud/ocdav/get.go @@ -36,6 +36,7 @@ import ( "github.com/cs3org/reva/pkg/rhttp" rtrace "github.com/cs3org/reva/pkg/trace" "github.com/cs3org/reva/pkg/utils" + "github.com/cs3org/reva/pkg/utils/resourceid" "github.com/rs/zerolog" ) @@ -126,7 +127,7 @@ func (s *svc) handleGet(ctx context.Context, w http.ResponseWriter, r *http.Requ w.Header().Set(HeaderContentDisposistion, "attachment; filename*=UTF-8''"+ path.Base(r.RequestURI)+"; filename=\""+path.Base(r.RequestURI)+"\"") w.Header().Set(HeaderETag, info.Etag) - w.Header().Set(HeaderOCFileID, wrapResourceID(info.Id)) + w.Header().Set(HeaderOCFileID, resourceid.OwnCloudResourceIDWrap(info.Id)) w.Header().Set(HeaderOCETag, info.Etag) t := utils.TSToTime(info.Mtime).UTC() lastModifiedString := t.Format(time.RFC1123Z) diff --git a/internal/http/services/owncloud/ocdav/head.go b/internal/http/services/owncloud/ocdav/head.go index 7b3094595b..7acdca6a8e 100644 --- a/internal/http/services/owncloud/ocdav/head.go +++ b/internal/http/services/owncloud/ocdav/head.go @@ -28,6 +28,7 @@ import ( "time" rtrace "github.com/cs3org/reva/pkg/trace" + "github.com/cs3org/reva/pkg/utils/resourceid" rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" @@ -73,7 +74,7 @@ func (s *svc) handleHead(ctx context.Context, w http.ResponseWriter, r *http.Req info := res.Info w.Header().Set(HeaderContentType, info.MimeType) w.Header().Set(HeaderETag, info.Etag) - w.Header().Set(HeaderOCFileID, wrapResourceID(info.Id)) + w.Header().Set(HeaderOCFileID, resourceid.OwnCloudResourceIDWrap(info.Id)) w.Header().Set(HeaderOCETag, info.Etag) if info.Checksum != nil { w.Header().Set(HeaderOCChecksum, fmt.Sprintf("%s:%s", strings.ToUpper(string(storageprovider.GRPC2PKGXS(info.Checksum.Type))), info.Checksum.Sum)) diff --git a/internal/http/services/owncloud/ocdav/meta.go b/internal/http/services/owncloud/ocdav/meta.go index f40baa2396..7aaffea142 100644 --- a/internal/http/services/owncloud/ocdav/meta.go +++ b/internal/http/services/owncloud/ocdav/meta.go @@ -22,6 +22,7 @@ import ( "net/http" "github.com/cs3org/reva/pkg/rhttp/router" + "github.com/cs3org/reva/pkg/utils/resourceid" ) // MetaHandler handles meta requests @@ -45,7 +46,7 @@ func (h *MetaHandler) Handler(s *svc) http.Handler { return } - did := unwrap(id) + did := resourceid.OwnCloudResourceIDUnwrap(id) var head string head, r.URL.Path = router.ShiftPath(r.URL.Path) diff --git a/internal/http/services/owncloud/ocdav/move.go b/internal/http/services/owncloud/ocdav/move.go index 376f57d72b..53d3852f8c 100644 --- a/internal/http/services/owncloud/ocdav/move.go +++ b/internal/http/services/owncloud/ocdav/move.go @@ -30,6 +30,7 @@ import ( "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/rhttp/router" rtrace "github.com/cs3org/reva/pkg/trace" + "github.com/cs3org/reva/pkg/utils/resourceid" "github.com/rs/zerolog" ) @@ -259,7 +260,7 @@ func (s *svc) handleMove(ctx context.Context, w http.ResponseWriter, r *http.Req info := dstStatRes.Info w.Header().Set(HeaderContentType, info.MimeType) w.Header().Set(HeaderETag, info.Etag) - w.Header().Set(HeaderOCFileID, wrapResourceID(info.Id)) + w.Header().Set(HeaderOCFileID, resourceid.OwnCloudResourceIDWrap(info.Id)) w.Header().Set(HeaderOCETag, info.Etag) w.WriteHeader(successCode) } diff --git a/internal/http/services/owncloud/ocdav/ocdav.go b/internal/http/services/owncloud/ocdav/ocdav.go index 27d11407b2..e53daa7ccf 100644 --- a/internal/http/services/owncloud/ocdav/ocdav.go +++ b/internal/http/services/owncloud/ocdav/ocdav.go @@ -20,7 +20,6 @@ package ocdav import ( "context" - "encoding/base64" "fmt" "net/http" "net/url" @@ -28,11 +27,9 @@ import ( "regexp" "strings" "time" - "unicode/utf8" gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1" userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" - provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/cs3org/reva/pkg/appctx" ctxpkg "github.com/cs3org/reva/pkg/ctx" "github.com/cs3org/reva/pkg/errtypes" @@ -53,8 +50,6 @@ type ctxKey int const ( ctxKeyBaseURI ctxKey = iota - - idDelimiter string = ":" ) var ( @@ -271,39 +266,6 @@ func applyLayout(ctx context.Context, ns string, useLoggedInUserNS bool, request return templates.WithUser(u, ns) } -func wrapResourceID(r *provider.ResourceId) string { - return wrap(r.StorageId, r.OpaqueId) -} - -// The fileID must be encoded -// - XML safe, because it is going to be used in the propfind result -// - url safe, because the id might be used in a url, eg. the /dav/meta nodes -// which is why we base64 encode it -func wrap(sid string, oid string) string { - return base64.URLEncoding.EncodeToString([]byte(sid + idDelimiter + oid)) -} - -func unwrap(rid string) *provider.ResourceId { - decodedID, err := base64.URLEncoding.DecodeString(rid) - if err != nil { - return nil - } - - parts := strings.SplitN(string(decodedID), idDelimiter, 2) - if len(parts) != 2 { - return nil - } - - if !utf8.ValidString(parts[0]) || !utf8.ValidString(parts[1]) { - return nil - } - - return &provider.ResourceId{ - StorageId: parts[0], - OpaqueId: parts[1], - } -} - func addAccessHeaders(w http.ResponseWriter, r *http.Request) { headers := w.Header() // the webdav api is accessible from anywhere diff --git a/internal/http/services/owncloud/ocdav/ocdav_test.go b/internal/http/services/owncloud/ocdav/ocdav_test.go index 24326cb9f9..7d0de23167 100644 --- a/internal/http/services/owncloud/ocdav/ocdav_test.go +++ b/internal/http/services/owncloud/ocdav/ocdav_test.go @@ -25,7 +25,7 @@ import ( "testing" providerv1beta1 "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" - "github.com/cs3org/reva/pkg/utils" + "github.com/cs3org/reva/pkg/utils/resourceid" ) /* @@ -41,69 +41,15 @@ func BenchmarkEncodePath(b *testing.B) { } } -func BenchmarkWrap(b *testing.B) { - for i := 0; i < b.N; i++ { - _ = wrap("storageid", "opaqueid") - } -} - -func TestWrap(t *testing.T) { - expected := "c3RvcmFnZWlkOm9wYXF1ZWlk" - wrapped := wrap("storageid", "opaqueid") - - if wrapped != expected { - t.Errorf("wrapped id doesn't have the expected format: got %s expected %s", wrapped, expected) - } -} - func TestWrapResourceID(t *testing.T) { expected := "c3RvcmFnZWlkOm9wYXF1ZWlk" - wrapped := wrapResourceID(&providerv1beta1.ResourceId{StorageId: "storageid", OpaqueId: "opaqueid"}) + wrapped := resourceid.OwnCloudResourceIDWrap(&providerv1beta1.ResourceId{StorageId: "storageid", OpaqueId: "opaqueid"}) if wrapped != expected { t.Errorf("wrapped id doesn't have the expected format: got %s expected %s", wrapped, expected) } } -func BenchmarkUnwrap(b *testing.B) { - for i := 0; i < b.N; i++ { - _ = unwrap("c3RvcmFnZWlkOm9wYXF1ZWlk") - } -} - -func TestUnwrap(t *testing.T) { - tests := []struct { - input string - expected *providerv1beta1.ResourceId - }{ - { - "c3RvcmFnZWlkOm9wYXF1ZWlk", - &providerv1beta1.ResourceId{StorageId: "storageid", OpaqueId: "opaqueid"}, - }, - { - "", - nil, - }, - { - "c", - nil, - }, - } - - for _, tt := range tests { - rid := unwrap(tt.input) - - if tt.expected == nil { - if rid != nil { - t.Errorf("Expected unwrap to return nil, got %v", rid) - } - } else if !utils.ResourceIDEqual(rid, tt.expected) { - t.Error("StorageID or OpaqueID doesn't match") - } - } - -} - func TestExtractDestination(t *testing.T) { expected := "/dst" request := httptest.NewRequest(http.MethodGet, "https://example.org/remote.php/dav/src", nil) diff --git a/internal/http/services/owncloud/ocdav/propfind.go b/internal/http/services/owncloud/ocdav/propfind.go index eb0e0b197f..a9adf2848d 100644 --- a/internal/http/services/owncloud/ocdav/propfind.go +++ b/internal/http/services/owncloud/ocdav/propfind.go @@ -44,6 +44,7 @@ import ( "github.com/cs3org/reva/pkg/publicshare" rtrace "github.com/cs3org/reva/pkg/trace" "github.com/cs3org/reva/pkg/utils" + "github.com/cs3org/reva/pkg/utils/resourceid" "github.com/rs/zerolog" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/codes" @@ -550,7 +551,7 @@ func (s *svc) mdToPropResponse(ctx context.Context, pf *propfindXML, md *provide // return all known properties if md.Id != nil { - id := wrapResourceID(md.Id) + id := resourceid.OwnCloudResourceIDWrap(md.Id) propstatOK.Prop = append(propstatOK.Prop, s.newProp("oc:id", id), s.newProp("oc:fileid", id), @@ -649,13 +650,13 @@ func (s *svc) mdToPropResponse(ctx context.Context, pf *propfindXML, md *provide // I tested the desktop client and phoenix to annotate which properties are requestted, see below cases case "fileid": // phoenix only if md.Id != nil { - propstatOK.Prop = append(propstatOK.Prop, s.newProp("oc:fileid", wrapResourceID(md.Id))) + propstatOK.Prop = append(propstatOK.Prop, s.newProp("oc:fileid", resourceid.OwnCloudResourceIDWrap(md.Id))) } else { propstatNotFound.Prop = append(propstatNotFound.Prop, s.newProp("oc:fileid", "")) } case "id": // desktop client only if md.Id != nil { - propstatOK.Prop = append(propstatOK.Prop, s.newProp("oc:id", wrapResourceID(md.Id))) + propstatOK.Prop = append(propstatOK.Prop, s.newProp("oc:id", resourceid.OwnCloudResourceIDWrap(md.Id))) } else { propstatNotFound.Prop = append(propstatNotFound.Prop, s.newProp("oc:id", "")) } diff --git a/internal/http/services/owncloud/ocdav/put.go b/internal/http/services/owncloud/ocdav/put.go index f28a423404..1114d604f6 100644 --- a/internal/http/services/owncloud/ocdav/put.go +++ b/internal/http/services/owncloud/ocdav/put.go @@ -36,6 +36,7 @@ import ( "github.com/cs3org/reva/pkg/storage/utils/chunking" rtrace "github.com/cs3org/reva/pkg/trace" "github.com/cs3org/reva/pkg/utils" + "github.com/cs3org/reva/pkg/utils/resourceid" "github.com/rs/zerolog" ) @@ -314,7 +315,7 @@ func (s *svc) handlePut(ctx context.Context, w http.ResponseWriter, r *http.Requ w.Header().Add(HeaderContentType, newInfo.MimeType) w.Header().Set(HeaderETag, newInfo.Etag) - w.Header().Set(HeaderOCFileID, wrapResourceID(newInfo.Id)) + w.Header().Set(HeaderOCFileID, resourceid.OwnCloudResourceIDWrap(newInfo.Id)) w.Header().Set(HeaderOCETag, newInfo.Etag) t := utils.TSToTime(newInfo.Mtime).UTC() lastModifiedString := t.Format(time.RFC1123Z) diff --git a/internal/http/services/owncloud/ocdav/trashbin.go b/internal/http/services/owncloud/ocdav/trashbin.go index 5d7425d1bd..7d27aa0964 100644 --- a/internal/http/services/owncloud/ocdav/trashbin.go +++ b/internal/http/services/owncloud/ocdav/trashbin.go @@ -30,6 +30,7 @@ import ( "time" rtrace "github.com/cs3org/reva/pkg/trace" + "github.com/cs3org/reva/pkg/utils/resourceid" userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" @@ -580,7 +581,7 @@ func (h *TrashbinHandler) restore(w http.ResponseWriter, r *http.Request, s *svc info := dstStatRes.Info w.Header().Set(HeaderContentType, info.MimeType) w.Header().Set(HeaderETag, info.Etag) - w.Header().Set(HeaderOCFileID, wrapResourceID(info.Id)) + w.Header().Set(HeaderOCFileID, resourceid.OwnCloudResourceIDWrap(info.Id)) w.Header().Set(HeaderOCETag, info.Etag) w.WriteHeader(successCode) diff --git a/internal/http/services/owncloud/ocdav/tus.go b/internal/http/services/owncloud/ocdav/tus.go index 8dea0a0fce..b795d482d4 100644 --- a/internal/http/services/owncloud/ocdav/tus.go +++ b/internal/http/services/owncloud/ocdav/tus.go @@ -36,6 +36,7 @@ import ( "github.com/cs3org/reva/pkg/rhttp" rtrace "github.com/cs3org/reva/pkg/trace" "github.com/cs3org/reva/pkg/utils" + "github.com/cs3org/reva/pkg/utils/resourceid" "github.com/rs/zerolog" tusd "github.com/tus/tusd/pkg/handler" ) @@ -313,7 +314,7 @@ func (s *svc) handleTusPost(ctx context.Context, w http.ResponseWriter, r *http. ) w.Header().Set(HeaderContentType, info.MimeType) - w.Header().Set(HeaderOCFileID, wrapResourceID(info.Id)) + w.Header().Set(HeaderOCFileID, resourceid.OwnCloudResourceIDWrap(info.Id)) w.Header().Set(HeaderOCETag, info.Etag) w.Header().Set(HeaderETag, info.Etag) w.Header().Set(HeaderOCPermissions, permissions) diff --git a/internal/http/services/owncloud/ocdav/versions.go b/internal/http/services/owncloud/ocdav/versions.go index ad6620ccd9..75467dd39e 100644 --- a/internal/http/services/owncloud/ocdav/versions.go +++ b/internal/http/services/owncloud/ocdav/versions.go @@ -24,6 +24,7 @@ import ( "path" rtrace "github.com/cs3org/reva/pkg/trace" + "github.com/cs3org/reva/pkg/utils/resourceid" rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" @@ -53,7 +54,7 @@ func (h *VersionsHandler) Handler(s *svc, rid *provider.ResourceId) http.Handler } // baseURI is encoded as part of the response payload in href field - baseURI := path.Join(ctx.Value(ctxKeyBaseURI).(string), wrapResourceID(rid)) + baseURI := path.Join(ctx.Value(ctxKeyBaseURI).(string), resourceid.OwnCloudResourceIDWrap(rid)) ctx = context.WithValue(ctx, ctxKeyBaseURI, baseURI) r = r.WithContext(ctx) diff --git a/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/shares.go b/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/shares.go index f1937658ca..53c859c7c0 100644 --- a/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/shares.go +++ b/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/shares.go @@ -21,7 +21,6 @@ package shares import ( "bytes" "context" - "encoding/base64" "encoding/json" "fmt" "mime" @@ -55,6 +54,7 @@ import ( "github.com/cs3org/reva/pkg/share/cache" "github.com/cs3org/reva/pkg/share/cache/registry" "github.com/cs3org/reva/pkg/utils" + "github.com/cs3org/reva/pkg/utils/resourceid" "github.com/pkg/errors" ) @@ -119,7 +119,7 @@ func (h *Handler) startCacheWarmup(c cache.Warmup) { return } for _, r := range infos { - key := wrapResourceID(r.Id) + key := resourceid.OwnCloudResourceIDWrap(r.Id) _ = h.resourceInfoCache.SetWithExpire(key, r, h.resourceInfoCacheTTL) } } @@ -856,18 +856,6 @@ func (h *Handler) addFilters(w http.ResponseWriter, r *http.Request, prefix stri return collaborationFilters, linkFilters, nil } -func wrapResourceID(r *provider.ResourceId) string { - return wrap(r.StorageId, r.OpaqueId) -} - -// The fileID must be encoded -// - XML safe, because it is going to be used in the propfind result -// - url safe, because the id might be used in a url, eg. the /dav/meta nodes -// which is why we base64 encode it -func wrap(sid string, oid string) string { - return base64.URLEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s", sid, oid))) -} - func (h *Handler) addFileInfo(ctx context.Context, s *conversions.ShareData, info *provider.ResourceInfo) error { log := appctx.GetLogger(ctx) if info != nil { @@ -880,7 +868,7 @@ func (h *Handler) addFileInfo(ctx context.Context, s *conversions.ShareData, inf s.MimeType = parsedMt // TODO STime: &types.Timestamp{Seconds: info.Mtime.Seconds, Nanos: info.Mtime.Nanos}, // TODO Storage: int - s.ItemSource = wrapResourceID(info.Id) + s.ItemSource = resourceid.OwnCloudResourceIDWrap(info.Id) s.FileSource = s.ItemSource switch { case h.sharePrefix == "/": @@ -1041,7 +1029,7 @@ func (h *Handler) getResourceInfoByPath(ctx context.Context, client gateway.Gate } func (h *Handler) getResourceInfoByID(ctx context.Context, client gateway.GatewayAPIClient, id *provider.ResourceId) (*provider.ResourceInfo, *rpc.Status, error) { - return h.getResourceInfo(ctx, client, wrapResourceID(id), &provider.Reference{ResourceId: id}) + return h.getResourceInfo(ctx, client, resourceid.OwnCloudResourceIDWrap(id), &provider.Reference{ResourceId: id}) } // getResourceInfo retrieves the resource info to a target. diff --git a/pkg/utils/resourceid/owncloud.go b/pkg/utils/resourceid/owncloud.go new file mode 100644 index 0000000000..39abc460a4 --- /dev/null +++ b/pkg/utils/resourceid/owncloud.go @@ -0,0 +1,77 @@ +// Copyright 2018-2021 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package resourceid + +import ( + "encoding/base64" + "errors" + "strings" + "unicode/utf8" + + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" +) + +const ( + idDelimiter string = ":" +) + +// OwnCloudResourceIDUnwrap returns the wrapped resource id +// by OwnCloudResourceIDWrap and returns nil if not possible +func OwnCloudResourceIDUnwrap(rid string) *provider.ResourceId { + id, err := unwrap(rid) + if err != nil { + return nil + } + return id +} + +func unwrap(rid string) (*provider.ResourceId, error) { + decodedID, err := base64.URLEncoding.DecodeString(rid) + if err != nil { + return nil, err + } + + parts := strings.SplitN(string(decodedID), idDelimiter, 2) + if len(parts) != 2 { + return nil, errors.New("could not find two parts with given delimiter") + } + + if !utf8.ValidString(parts[0]) || !utf8.ValidString(parts[1]) { + return nil, errors.New("invalid utf8 string found") + } + + return &provider.ResourceId{ + StorageId: parts[0], + OpaqueId: parts[1], + }, nil +} + +// OwnCloudResourceIDWrap wraps a resource id into a xml safe string +// which can then be passed to the outside world +func OwnCloudResourceIDWrap(r *provider.ResourceId) string { + return wrap(r.StorageId, r.OpaqueId) +} + +// The fileID must be encoded +// - XML safe, because it is going to be used in the propfind result +// - url safe, because the id might be used in a url, eg. the /dav/meta nodes +// which is why we base64 encode it +func wrap(sid string, oid string) string { + return base64.URLEncoding.EncodeToString([]byte(sid + idDelimiter + oid)) +} diff --git a/pkg/utils/resourceid/owncloud_test.go b/pkg/utils/resourceid/owncloud_test.go new file mode 100644 index 0000000000..5fc45af94a --- /dev/null +++ b/pkg/utils/resourceid/owncloud_test.go @@ -0,0 +1,88 @@ +// Copyright 2021 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. +package resourceid + +import ( + "testing" + + providerv1beta1 "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/cs3org/reva/pkg/utils" +) + +func BenchmarkWrap(b *testing.B) { + for i := 0; i < b.N; i++ { + _ = wrap("storageid", "opaqueid") + } +} + +func TestWrap(t *testing.T) { + expected := "c3RvcmFnZWlkOm9wYXF1ZWlk" + wrapped := wrap("storageid", "opaqueid") + + if wrapped != expected { + t.Errorf("wrapped id doesn't have the expected format: got %s expected %s", wrapped, expected) + } +} + +func TestWrapResourceID(t *testing.T) { + expected := "c3RvcmFnZWlkOm9wYXF1ZWlk" + wrapped := OwnCloudResourceIDWrap(&providerv1beta1.ResourceId{StorageId: "storageid", OpaqueId: "opaqueid"}) + + if wrapped != expected { + t.Errorf("wrapped id doesn't have the expected format: got %s expected %s", wrapped, expected) + } +} + +func BenchmarkUnwrap(b *testing.B) { + for i := 0; i < b.N; i++ { + _, _ = unwrap("c3RvcmFnZWlkOm9wYXF1ZWlk") + } +} + +func TestUnwrap(t *testing.T) { + tests := []struct { + input string + expected *providerv1beta1.ResourceId + }{ + { + "c3RvcmFnZWlkOm9wYXF1ZWlk", + &providerv1beta1.ResourceId{StorageId: "storageid", OpaqueId: "opaqueid"}, + }, + { + "", + nil, + }, + { + "c", + nil, + }, + } + + for _, tt := range tests { + rid := OwnCloudResourceIDUnwrap(tt.input) + + if tt.expected == nil { + if rid != nil { + t.Errorf("Expected unwrap to return nil, got %v", rid) + } + } else if !utils.ResourceIDEqual(rid, tt.expected) { + t.Error("StorageID or OpaqueID doesn't match") + } + } + +} From 2f727fc050a596458f6d96406a513e8bb053002a Mon Sep 17 00:00:00 2001 From: Ishank Arora Date: Thu, 9 Dec 2021 19:36:07 +0530 Subject: [PATCH 11/58] Parse URL path to determine file name (#2346) --- .../unreleased/fix-content-disposition.md | 3 +- .../auth/manager/oidcmapping/_index.md | 66 +++++++++++++++++++ internal/http/services/owncloud/ocdav/get.go | 2 +- 3 files changed, 69 insertions(+), 2 deletions(-) create mode 100644 docs/content/en/docs/config/packages/auth/manager/oidcmapping/_index.md diff --git a/changelog/unreleased/fix-content-disposition.md b/changelog/unreleased/fix-content-disposition.md index 58f126a7ac..da3822ceb1 100644 --- a/changelog/unreleased/fix-content-disposition.md +++ b/changelog/unreleased/fix-content-disposition.md @@ -2,4 +2,5 @@ Bugfix: Fix content disposition header for public links files https://github.com/cs3org/reva/pull/2303 https://github.com/cs3org/reva/pull/2297 -https://github.com/cs3org/reva/pull/2332 \ No newline at end of file +https://github.com/cs3org/reva/pull/2332 +https://github.com/cs3org/reva/pull/2346 \ No newline at end of file diff --git a/docs/content/en/docs/config/packages/auth/manager/oidcmapping/_index.md b/docs/content/en/docs/config/packages/auth/manager/oidcmapping/_index.md new file mode 100644 index 0000000000..a7309eb3e1 --- /dev/null +++ b/docs/content/en/docs/config/packages/auth/manager/oidcmapping/_index.md @@ -0,0 +1,66 @@ +--- +title: "oidcmapping" +linkTitle: "oidcmapping" +weight: 10 +description: > + Configuration for the oidcmapping service +--- + +# _struct: config_ + +{{% dir name="insecure" type="bool" default=false %}} +Whether to skip certificate checks when sending requests. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/auth/manager/oidcmapping/oidcmapping.go#L57) +{{< highlight toml >}} +[auth.manager.oidcmapping] +insecure = false +{{< /highlight >}} +{{% /dir %}} + +{{% dir name="issuer" type="string" default="" %}} +The issuer of the OIDC token. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/auth/manager/oidcmapping/oidcmapping.go#L58) +{{< highlight toml >}} +[auth.manager.oidcmapping] +issuer = "" +{{< /highlight >}} +{{% /dir %}} + +{{% dir name="id_claim" type="string" default="sub" %}} +The claim containing the ID of the user. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/auth/manager/oidcmapping/oidcmapping.go#L59) +{{< highlight toml >}} +[auth.manager.oidcmapping] +id_claim = "sub" +{{< /highlight >}} +{{% /dir %}} + +{{% dir name="uid_claim" type="string" default="" %}} +The claim containing the UID of the user. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/auth/manager/oidcmapping/oidcmapping.go#L60) +{{< highlight toml >}} +[auth.manager.oidcmapping] +uid_claim = "" +{{< /highlight >}} +{{% /dir %}} + +{{% dir name="gid_claim" type="string" default="" %}} +The claim containing the GID of the user. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/auth/manager/oidcmapping/oidcmapping.go#L61) +{{< highlight toml >}} +[auth.manager.oidcmapping] +gid_claim = "" +{{< /highlight >}} +{{% /dir %}} + +{{% dir name="userprovidersvc" type="string" default="" %}} +The endpoint at which the GRPC userprovider is exposed. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/auth/manager/oidcmapping/oidcmapping.go#L62) +{{< highlight toml >}} +[auth.manager.oidcmapping] +userprovidersvc = "" +{{< /highlight >}} +{{% /dir %}} + +{{% dir name="usersmapping" type="string" default="" %}} + The OIDC users mapping file path [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/auth/manager/oidcmapping/oidcmapping.go#L63) +{{< highlight toml >}} +[auth.manager.oidcmapping] +usersmapping = "" +{{< /highlight >}} +{{% /dir %}} + diff --git a/internal/http/services/owncloud/ocdav/get.go b/internal/http/services/owncloud/ocdav/get.go index a812365846..b93230ab51 100644 --- a/internal/http/services/owncloud/ocdav/get.go +++ b/internal/http/services/owncloud/ocdav/get.go @@ -125,7 +125,7 @@ func (s *svc) handleGet(ctx context.Context, w http.ResponseWriter, r *http.Requ w.Header().Set(HeaderContentType, info.MimeType) w.Header().Set(HeaderContentDisposistion, "attachment; filename*=UTF-8''"+ - path.Base(r.RequestURI)+"; filename=\""+path.Base(r.RequestURI)+"\"") + path.Base(r.URL.Path)+"; filename=\""+path.Base(r.URL.Path)+"\"") w.Header().Set(HeaderETag, info.Etag) w.Header().Set(HeaderOCFileID, resourceid.OwnCloudResourceIDWrap(info.Id)) w.Header().Set(HeaderOCETag, info.Etag) From d54df2f488df3560d332a68bcc70ee175954140e Mon Sep 17 00:00:00 2001 From: Ishank Arora Date: Thu, 9 Dec 2021 19:41:11 +0530 Subject: [PATCH 12/58] v1.17.0 --- CHANGELOG.md | 121 ++++++++ RELEASE_DATE | 2 +- VERSION | 2 +- .../app-abspath.md | 0 .../app-provider-new-file.md | 0 .../fix-content-disposition.md | 0 .../fix-propfind-sharetype.md | 0 .../fix-public-link-wopi.md | 0 .../fix-share-recyclebin.md | 0 .../fix-skip-early-empty-file-creation.md | 0 .../iodc-escape-auth.md | 0 .../ocs-user-type.md | 0 .../public-link-propfind.md | 0 .../space-membership.md | 0 .../sysacl-from-xattr.md | 0 .../tus-upload-with-transfer-token-only.md | 0 .../user-claim-fallback.md | 0 changelog/NOTE.md | 288 +++++------------- .../en/docs/changelog/1.17.0/_index.md | 130 ++++++++ 19 files changed, 321 insertions(+), 222 deletions(-) rename changelog/{unreleased => 1.17.0_2021-12-09}/app-abspath.md (100%) rename changelog/{unreleased => 1.17.0_2021-12-09}/app-provider-new-file.md (100%) rename changelog/{unreleased => 1.17.0_2021-12-09}/fix-content-disposition.md (100%) rename changelog/{unreleased => 1.17.0_2021-12-09}/fix-propfind-sharetype.md (100%) rename changelog/{unreleased => 1.17.0_2021-12-09}/fix-public-link-wopi.md (100%) rename changelog/{unreleased => 1.17.0_2021-12-09}/fix-share-recyclebin.md (100%) rename changelog/{unreleased => 1.17.0_2021-12-09}/fix-skip-early-empty-file-creation.md (100%) rename changelog/{unreleased => 1.17.0_2021-12-09}/iodc-escape-auth.md (100%) rename changelog/{unreleased => 1.17.0_2021-12-09}/ocs-user-type.md (100%) rename changelog/{unreleased => 1.17.0_2021-12-09}/public-link-propfind.md (100%) rename changelog/{unreleased => 1.17.0_2021-12-09}/space-membership.md (100%) rename changelog/{unreleased => 1.17.0_2021-12-09}/sysacl-from-xattr.md (100%) rename changelog/{unreleased => 1.17.0_2021-12-09}/tus-upload-with-transfer-token-only.md (100%) rename changelog/{unreleased => 1.17.0_2021-12-09}/user-claim-fallback.md (100%) create mode 100644 docs/content/en/docs/changelog/1.17.0/_index.md diff --git a/CHANGELOG.md b/CHANGELOG.md index c012445f1e..825b70bcaf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,124 @@ +Changelog for reva 1.17.0 (2021-12-09) +======================================= + +The following sections list the changes in reva 1.17.0 relevant to +reva users. The changes are ordered by importance. + +Summary +------- + + * Fix #2305: Make sure /app/new takes `target` as absolute path + * Fix #2303: Fix content disposition header for public links files + * Fix #2316: Fix the share types in propfinds + * Fix #2803: Fix app provider for editor public links + * Fix #2298: Remove share refs from trashbin + * Fix #2309: Remove early finish for zero byte file uploads + * Fix #1941: Fix TUS uploads with transfer token only + * Chg #2210: Fix app provider new file creation and improved error codes + * Enh #2217: OIDC auth driver for ESCAPE IAM + * Enh #2256: Return user type in the response of the ocs GET user call + * Enh #2315: Add new attributes to public link propfinds + * Enh #2740: Implement space membership endpoints + * Enh #2252: Add the xattr sys.acl to SysACL (eosgrpc) + * Enh #2314: OIDC: fallback if IDP doesn't provide "preferred_username" claim + +Details +------- + + * Bugfix #2305: Make sure /app/new takes `target` as absolute path + + A mini-PR to make the `target` parameter absolute (by prepending `/` if missing). + + https://github.com/cs3org/reva/pull/2305 + + * Bugfix #2303: Fix content disposition header for public links files + + https://github.com/cs3org/reva/pull/2303 + https://github.com/cs3org/reva/pull/2297 + https://github.com/cs3org/reva/pull/2332 + https://github.com/cs3org/reva/pull/2346 + + * Bugfix #2316: Fix the share types in propfinds + + The share types for public links were not correctly added to propfinds. + + https://github.com/cs3org/reva/pull/2316 + + * Bugfix #2803: Fix app provider for editor public links + + Fixed opening the app provider in public links with the editor permission. The app provider + failed to open the file in read write mode. + + https://github.com/owncloud/ocis/issues/2803 + https://github.com/cs3org/reva/pull/2310 + + * Bugfix #2298: Remove share refs from trashbin + + https://github.com/cs3org/reva/pull/2298 + + * Bugfix #2309: Remove early finish for zero byte file uploads + + We've fixed the upload of zero byte files by removing the early upload finishing mechanism. + + https://github.com/cs3org/reva/issues/2309 + https://github.com/owncloud/ocis/issues/2609 + + * Bugfix #1941: Fix TUS uploads with transfer token only + + TUS uploads had been stopped when the user JWT token expired, even if only the transfer token + should be validated. Now uploads will continue as intended. + + https://github.com/cs3org/reva/pull/1941 + + * Change #2210: Fix app provider new file creation and improved error codes + + We've fixed the behavior for the app provider when creating new files. Previously the app + provider would overwrite already existing files when creating a new file, this is now handled + and prevented. The new file endpoint accepted a path to a file, but this does not work for spaces. + Therefore we now use the resource id of the folder where the file should be created and a filename + to create the new file. Also the app provider returns more useful error codes in a lot of cases. + + https://github.com/cs3org/reva/pull/2210 + + * Enhancement #2217: OIDC auth driver for ESCAPE IAM + + This enhancement allows for oidc token authentication via the ESCAPE IAM service. + Authentication relies on mappings of ESCAPE IAM groups to REVA users. For a valid token, if at + the most one group from the groups claim is mapped to one REVA user, authentication can take + place. + + https://github.com/cs3org/reva/pull/2217 + + * Enhancement #2256: Return user type in the response of the ocs GET user call + + https://github.com/cs3org/reva/pull/2256 + + * Enhancement #2315: Add new attributes to public link propfinds + + Added a new property "oc:signature-auth" to public link propfinds. This is a necessary change + to be able to support archive downloads in password protected public links. + + https://github.com/cs3org/reva/pull/2315 + + * Enhancement #2740: Implement space membership endpoints + + Implemented endpoints to add and remove members to spaces. + + https://github.com/owncloud/ocis/issues/2740 + https://github.com/cs3org/reva/pull/2250 + + * Enhancement #2252: Add the xattr sys.acl to SysACL (eosgrpc) + + https://github.com/cs3org/reva/pull/2252 + + * Enhancement #2314: OIDC: fallback if IDP doesn't provide "preferred_username" claim + + Some IDPs don't support the "preferred_username" claim. Fallback to the "email" claim in that + case. + + https://github.com/cs3org/reva/pull/2314 + + Changelog for reva 1.16.0 (2021-11-19) ======================================= diff --git a/RELEASE_DATE b/RELEASE_DATE index 15e3b281a6..3b6b771b94 100644 --- a/RELEASE_DATE +++ b/RELEASE_DATE @@ -1 +1 @@ -2021-11-19 \ No newline at end of file +2021-12-09 \ No newline at end of file diff --git a/VERSION b/VERSION index 71bd5d9ee6..73d74673c7 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.16.0 \ No newline at end of file +1.17.0 \ No newline at end of file diff --git a/changelog/unreleased/app-abspath.md b/changelog/1.17.0_2021-12-09/app-abspath.md similarity index 100% rename from changelog/unreleased/app-abspath.md rename to changelog/1.17.0_2021-12-09/app-abspath.md diff --git a/changelog/unreleased/app-provider-new-file.md b/changelog/1.17.0_2021-12-09/app-provider-new-file.md similarity index 100% rename from changelog/unreleased/app-provider-new-file.md rename to changelog/1.17.0_2021-12-09/app-provider-new-file.md diff --git a/changelog/unreleased/fix-content-disposition.md b/changelog/1.17.0_2021-12-09/fix-content-disposition.md similarity index 100% rename from changelog/unreleased/fix-content-disposition.md rename to changelog/1.17.0_2021-12-09/fix-content-disposition.md diff --git a/changelog/unreleased/fix-propfind-sharetype.md b/changelog/1.17.0_2021-12-09/fix-propfind-sharetype.md similarity index 100% rename from changelog/unreleased/fix-propfind-sharetype.md rename to changelog/1.17.0_2021-12-09/fix-propfind-sharetype.md diff --git a/changelog/unreleased/fix-public-link-wopi.md b/changelog/1.17.0_2021-12-09/fix-public-link-wopi.md similarity index 100% rename from changelog/unreleased/fix-public-link-wopi.md rename to changelog/1.17.0_2021-12-09/fix-public-link-wopi.md diff --git a/changelog/unreleased/fix-share-recyclebin.md b/changelog/1.17.0_2021-12-09/fix-share-recyclebin.md similarity index 100% rename from changelog/unreleased/fix-share-recyclebin.md rename to changelog/1.17.0_2021-12-09/fix-share-recyclebin.md diff --git a/changelog/unreleased/fix-skip-early-empty-file-creation.md b/changelog/1.17.0_2021-12-09/fix-skip-early-empty-file-creation.md similarity index 100% rename from changelog/unreleased/fix-skip-early-empty-file-creation.md rename to changelog/1.17.0_2021-12-09/fix-skip-early-empty-file-creation.md diff --git a/changelog/unreleased/iodc-escape-auth.md b/changelog/1.17.0_2021-12-09/iodc-escape-auth.md similarity index 100% rename from changelog/unreleased/iodc-escape-auth.md rename to changelog/1.17.0_2021-12-09/iodc-escape-auth.md diff --git a/changelog/unreleased/ocs-user-type.md b/changelog/1.17.0_2021-12-09/ocs-user-type.md similarity index 100% rename from changelog/unreleased/ocs-user-type.md rename to changelog/1.17.0_2021-12-09/ocs-user-type.md diff --git a/changelog/unreleased/public-link-propfind.md b/changelog/1.17.0_2021-12-09/public-link-propfind.md similarity index 100% rename from changelog/unreleased/public-link-propfind.md rename to changelog/1.17.0_2021-12-09/public-link-propfind.md diff --git a/changelog/unreleased/space-membership.md b/changelog/1.17.0_2021-12-09/space-membership.md similarity index 100% rename from changelog/unreleased/space-membership.md rename to changelog/1.17.0_2021-12-09/space-membership.md diff --git a/changelog/unreleased/sysacl-from-xattr.md b/changelog/1.17.0_2021-12-09/sysacl-from-xattr.md similarity index 100% rename from changelog/unreleased/sysacl-from-xattr.md rename to changelog/1.17.0_2021-12-09/sysacl-from-xattr.md diff --git a/changelog/unreleased/tus-upload-with-transfer-token-only.md b/changelog/1.17.0_2021-12-09/tus-upload-with-transfer-token-only.md similarity index 100% rename from changelog/unreleased/tus-upload-with-transfer-token-only.md rename to changelog/1.17.0_2021-12-09/tus-upload-with-transfer-token-only.md diff --git a/changelog/unreleased/user-claim-fallback.md b/changelog/1.17.0_2021-12-09/user-claim-fallback.md similarity index 100% rename from changelog/unreleased/user-claim-fallback.md rename to changelog/1.17.0_2021-12-09/user-claim-fallback.md diff --git a/changelog/NOTE.md b/changelog/NOTE.md index 031de420a3..07b1251bac 100644 --- a/changelog/NOTE.md +++ b/changelog/NOTE.md @@ -1,273 +1,121 @@ -Changelog for reva 1.16.0 (2021-11-19) +Changelog for reva 1.17.0 (2021-12-09) ======================================= -The following sections list the changes in reva 1.16.0 relevant to +The following sections list the changes in reva 1.17.0 relevant to reva users. The changes are ordered by importance. Summary ------- - * Fix #2245: Don't announce search-files capability - * Fix #2247: Merge user ACLs from EOS to sys ACLs - * Fix #2279: Return the inode of the version folder for files when listing in EOS - * Fix #2294: Fix HTTP return code when path is invalid - * Fix #2231: Fix share permission on a single file in sql share driver (cbox pkg) - * Fix #2230: Fix open by default app and expose default app - * Fix #2265: Fix nil pointer exception when resolving members of a group (rest driver) - * Fix #1214: Fix restoring versions - * Fix #2254: Fix spaces propfind - * Fix #2260: Fix unset quota xattr on darwin - * Fix #5776: Enforce permissions in public share apps - * Fix #2767: Fix status code for WebDAV mkcol requests where an ancestor is missing - * Fix #2287: Add public link access via mount-ID:token/relative-path to the scope - * Fix #2244: Fix the permissions response for shared files in the cbox sql driver - * Enh #2219: Add virtual view tests - * Enh #2230: Add priority to app providers - * Enh #2258: Improved error messages from the AppProviders - * Enh #2119: Add authprovider owncloudsql - * Enh #2211: Enhance the cbox share sql driver to store accepted group shares - * Enh #2212: Filter root path according to the agent that makes the request - * Enh #2237: Skip get user call in eosfs in case previous ones also failed - * Enh #2266: Callback for the EOS UID cache to retry fetch for failed keys - * Enh #2215: Aggregrate resource info properties for virtual views - * Enh #2271: Revamp the favorite manager and add the cbox sql driver - * Enh #2248: Cache whether a user home was created or not - * Enh #2282: Return a proper NOT_FOUND error when a user or group is not found - * Enh #2268: Add the reverseproxy http service - * Enh #2207: Enable users to list all spaces - * Enh #2286: Add trace ID to middleware loggers - * Enh #2251: Mentix service inference - * Enh #2218: Allow filtering of mime types supported by app providers - * Enh #2213: Add public link share type to propfind response - * Enh #2253: Support the file editor role for public links - * Enh #2208: Reduce redundant stat calls when statting by resource ID - * Enh #2235: Specify a list of allowed folders/files to be archived - * Enh #2267: Restrict the paths where share creation is allowed + * Fix #2305: Make sure /app/new takes `target` as absolute path + * Fix #2303: Fix content disposition header for public links files + * Fix #2316: Fix the share types in propfinds + * Fix #2803: Fix app provider for editor public links + * Fix #2298: Remove share refs from trashbin + * Fix #2309: Remove early finish for zero byte file uploads + * Fix #1941: Fix TUS uploads with transfer token only + * Chg #2210: Fix app provider new file creation and improved error codes + * Enh #2217: OIDC auth driver for ESCAPE IAM + * Enh #2256: Return user type in the response of the ocs GET user call + * Enh #2315: Add new attributes to public link propfinds + * Enh #2740: Implement space membership endpoints * Enh #2252: Add the xattr sys.acl to SysACL (eosgrpc) - * Enh #2239: Update toml configs + * Enh #2314: OIDC: fallback if IDP doesn't provide "preferred_username" claim Details ------- - * Bugfix #2245: Don't announce search-files capability + * Bugfix #2305: Make sure /app/new takes `target` as absolute path - The `dav.reports` capability contained a `search-files` report which is currently not - implemented. We removed it from the defaults. + A mini-PR to make the `target` parameter absolute (by prepending `/` if missing). - https://github.com/cs3org/reva/pull/2245 + https://github.com/cs3org/reva/pull/2305 - * Bugfix #2247: Merge user ACLs from EOS to sys ACLs + * Bugfix #2303: Fix content disposition header for public links files - https://github.com/cs3org/reva/pull/2247 + https://github.com/cs3org/reva/pull/2303 + https://github.com/cs3org/reva/pull/2297 + https://github.com/cs3org/reva/pull/2332 + https://github.com/cs3org/reva/pull/2346 - * Bugfix #2279: Return the inode of the version folder for files when listing in EOS + * Bugfix #2316: Fix the share types in propfinds - https://github.com/cs3org/reva/pull/2279 + The share types for public links were not correctly added to propfinds. - * Bugfix #2294: Fix HTTP return code when path is invalid + https://github.com/cs3org/reva/pull/2316 - Before when a path was invalid, the archiver returned a 500 error code. Now this is fixed and - returns a 404 code. + * Bugfix #2803: Fix app provider for editor public links - https://github.com/cs3org/reva/pull/2294 + Fixed opening the app provider in public links with the editor permission. The app provider + failed to open the file in read write mode. - * Bugfix #2231: Fix share permission on a single file in sql share driver (cbox pkg) + https://github.com/owncloud/ocis/issues/2803 + https://github.com/cs3org/reva/pull/2310 - https://github.com/cs3org/reva/pull/2231 + * Bugfix #2298: Remove share refs from trashbin - * Bugfix #2230: Fix open by default app and expose default app + https://github.com/cs3org/reva/pull/2298 - We've fixed the open by default app name behaviour which previously only worked, if the default - app was configured by the provider address. We also now expose the default app on the - `/app/list` endpoint to clients. + * Bugfix #2309: Remove early finish for zero byte file uploads - https://github.com/cs3org/reva/issues/2230 - https://github.com/cs3org/cs3apis/pull/157 + We've fixed the upload of zero byte files by removing the early upload finishing mechanism. - * Bugfix #2265: Fix nil pointer exception when resolving members of a group (rest driver) + https://github.com/cs3org/reva/issues/2309 + https://github.com/owncloud/ocis/issues/2609 - https://github.com/cs3org/reva/pull/2265 + * Bugfix #1941: Fix TUS uploads with transfer token only - * Bugfix #1214: Fix restoring versions + TUS uploads had been stopped when the user JWT token expired, even if only the transfer token + should be validated. Now uploads will continue as intended. - Restoring a version would not remove that version from the version list. Now the behavior is - compatible to ownCloud 10. + https://github.com/cs3org/reva/pull/1941 - https://github.com/owncloud/ocis/issues/1214 - https://github.com/cs3org/reva/pull/2270 + * Change #2210: Fix app provider new file creation and improved error codes - * Bugfix #2254: Fix spaces propfind + We've fixed the behavior for the app provider when creating new files. Previously the app + provider would overwrite already existing files when creating a new file, this is now handled + and prevented. The new file endpoint accepted a path to a file, but this does not work for spaces. + Therefore we now use the resource id of the folder where the file should be created and a filename + to create the new file. Also the app provider returns more useful error codes in a lot of cases. - Fixed the deep listing of spaces. + https://github.com/cs3org/reva/pull/2210 - https://github.com/cs3org/reva/pull/2254 + * Enhancement #2217: OIDC auth driver for ESCAPE IAM - * Bugfix #2260: Fix unset quota xattr on darwin + This enhancement allows for oidc token authentication via the ESCAPE IAM service. + Authentication relies on mappings of ESCAPE IAM groups to REVA users. For a valid token, if at + the most one group from the groups claim is mapped to one REVA user, authentication can take + place. - Unset quota attributes were creating errors in the logfile on darwin. + https://github.com/cs3org/reva/pull/2217 - https://github.com/cs3org/reva/pull/2260 + * Enhancement #2256: Return user type in the response of the ocs GET user call - * Bugfix #5776: Enforce permissions in public share apps + https://github.com/cs3org/reva/pull/2256 - A receiver of a read-only public share could still edit files via apps like Collabora. These - changes enforce the share permissions in apps used on publicly shared resources. + * Enhancement #2315: Add new attributes to public link propfinds - https://github.com/owncloud/web/issues/5776 - https://github.com/owncloud/ocis/issues/2479 - https://github.com/cs3org/reva/pull/22142214 + Added a new property "oc:signature-auth" to public link propfinds. This is a necessary change + to be able to support archive downloads in password protected public links. - * Bugfix #2767: Fix status code for WebDAV mkcol requests where an ancestor is missing + https://github.com/cs3org/reva/pull/2315 - We've fixed the status code to 409 according to the WebDAV standard for MKCOL requests where an - ancestor is missing. Previously these requests would fail with an different error code (eg. - 500) because of storage driver limitations (eg. oCIS FS cannot handle recursive creation of - directories). + * Enhancement #2740: Implement space membership endpoints - https://github.com/owncloud/ocis/issues/2767 - https://github.com/cs3org/reva/pull/2293 + Implemented endpoints to add and remove members to spaces. - * Bugfix #2287: Add public link access via mount-ID:token/relative-path to the scope - - https://github.com/cs3org/reva/pull/2287 - - * Bugfix #2244: Fix the permissions response for shared files in the cbox sql driver - - https://github.com/cs3org/reva/pull/2244 - - * Enhancement #2219: Add virtual view tests - - https://github.com/cs3org/reva/pull/2219 - - * Enhancement #2230: Add priority to app providers - - Before the order of the list returned by the method FindProviders of app providers depended - from the order in which the app provider registered themselves. Now, it is possible to specify a - priority for each app provider, and even if an app provider re-register itself (for example - after a restart), the order is kept. - - https://github.com/cs3org/reva/pull/2230 - https://github.com/cs3org/cs3apis/pull/157 - https://github.com/cs3org/reva/pull/2263 - - * Enhancement #2258: Improved error messages from the AppProviders - - Some rather cryptic messages are now hidden to users, and some others are made more - user-friendly. Support for multiple locales is still missing and out of scope for now. - - https://github.com/cs3org/reva/pull/2258 - - * Enhancement #2119: Add authprovider owncloudsql - - We added an authprovider that can be configured to authenticate against an owncloud classic - mysql database. It verifies the password from the oc_users table. - - https://github.com/cs3org/reva/pull/2119 - - * Enhancement #2211: Enhance the cbox share sql driver to store accepted group shares - - https://github.com/cs3org/reva/pull/2211 - - * Enhancement #2212: Filter root path according to the agent that makes the request - - https://github.com/cs3org/reva/pull/2212 - - * Enhancement #2237: Skip get user call in eosfs in case previous ones also failed - - https://github.com/cs3org/reva/pull/2237 - - * Enhancement #2266: Callback for the EOS UID cache to retry fetch for failed keys - - https://github.com/cs3org/reva/pull/2266 - - * Enhancement #2215: Aggregrate resource info properties for virtual views - - https://github.com/cs3org/reva/pull/2215 - - * Enhancement #2271: Revamp the favorite manager and add the cbox sql driver - - https://github.com/cs3org/reva/pull/2271 - - * Enhancement #2248: Cache whether a user home was created or not - - Previously, on every call, we used to stat the user home to make sure that it existed. Now we cache - it for a given amount of time so as to avoid repeated calls. - - https://github.com/cs3org/reva/pull/2248 - - * Enhancement #2282: Return a proper NOT_FOUND error when a user or group is not found - - https://github.com/cs3org/reva/pull/2282 - - * Enhancement #2268: Add the reverseproxy http service - - This PR adds an HTTP service which does the job of authenticating incoming requests via the reva - middleware before forwarding them to the respective backends. This is useful for extensions - which do not have the auth mechanisms. - - https://github.com/cs3org/reva/pull/2268 - - * Enhancement #2207: Enable users to list all spaces - - Added a permission check if the user has the `list-all-spaces` permission. This enables users - to list all spaces, even those which they are not members of. - - https://github.com/cs3org/reva/pull/2207 - - * Enhancement #2286: Add trace ID to middleware loggers - - https://github.com/cs3org/reva/pull/2286 - - * Enhancement #2251: Mentix service inference - - Previously, 4 different services per site had to be created in the GOCDB. This PR removes this - redundancy by infering all endpoints from a single service entity, making site - administration a lot easier. - - https://github.com/cs3org/reva/pull/2251 - - * Enhancement #2218: Allow filtering of mime types supported by app providers - - https://github.com/cs3org/reva/pull/2218 - - * Enhancement #2213: Add public link share type to propfind response - - Added share type for public links to propfind responses. - - https://github.com/cs3org/reva/pull/2213 - https://github.com/cs3org/reva/pull/2257 - - * Enhancement #2253: Support the file editor role for public links - - https://github.com/cs3org/reva/pull/2253 - - * Enhancement #2208: Reduce redundant stat calls when statting by resource ID - - https://github.com/cs3org/reva/pull/2208 - - * Enhancement #2235: Specify a list of allowed folders/files to be archived - - Adds a configuration to the archiver service in order to specify a list of folders (as regex) - that can be archived. - - https://github.com/cs3org/reva/pull/2235 - - * Enhancement #2267: Restrict the paths where share creation is allowed - - This PR limits share creation to certain specified paths. These can be useful when users have - access to global spaces and virtual views but these should not be sharable. - - https://github.com/cs3org/reva/pull/2267 + https://github.com/owncloud/ocis/issues/2740 + https://github.com/cs3org/reva/pull/2250 * Enhancement #2252: Add the xattr sys.acl to SysACL (eosgrpc) https://github.com/cs3org/reva/pull/2252 - * Enhancement #2239: Update toml configs + * Enhancement #2314: OIDC: fallback if IDP doesn't provide "preferred_username" claim - We updated the local and drone configurations, cleanad up the example configs and removed the - reva gen subcommand which was generating outdated config. + Some IDPs don't support the "preferred_username" claim. Fallback to the "email" claim in that + case. - https://github.com/cs3org/reva/pull/2239 + https://github.com/cs3org/reva/pull/2314 diff --git a/docs/content/en/docs/changelog/1.17.0/_index.md b/docs/content/en/docs/changelog/1.17.0/_index.md new file mode 100644 index 0000000000..1e55092ca1 --- /dev/null +++ b/docs/content/en/docs/changelog/1.17.0/_index.md @@ -0,0 +1,130 @@ + +--- +title: "v1.17.0" +linkTitle: "v1.17.0" +weight: 40 +description: > + Changelog for Reva v1.17.0 (2021-12-09) +--- + +Changelog for reva 1.17.0 (2021-12-09) +======================================= + +The following sections list the changes in reva 1.17.0 relevant to +reva users. The changes are ordered by importance. + +Summary +------- + + * Fix #2305: Make sure /app/new takes `target` as absolute path + * Fix #2303: Fix content disposition header for public links files + * Fix #2316: Fix the share types in propfinds + * Fix #2803: Fix app provider for editor public links + * Fix #2298: Remove share refs from trashbin + * Fix #2309: Remove early finish for zero byte file uploads + * Fix #1941: Fix TUS uploads with transfer token only + * Chg #2210: Fix app provider new file creation and improved error codes + * Enh #2217: OIDC auth driver for ESCAPE IAM + * Enh #2256: Return user type in the response of the ocs GET user call + * Enh #2315: Add new attributes to public link propfinds + * Enh #2740: Implement space membership endpoints + * Enh #2252: Add the xattr sys.acl to SysACL (eosgrpc) + * Enh #2314: OIDC: fallback if IDP doesn't provide "preferred_username" claim + +Details +------- + + * Bugfix #2305: Make sure /app/new takes `target` as absolute path + + A mini-PR to make the `target` parameter absolute (by prepending `/` if missing). + + https://github.com/cs3org/reva/pull/2305 + + * Bugfix #2303: Fix content disposition header for public links files + + https://github.com/cs3org/reva/pull/2303 + https://github.com/cs3org/reva/pull/2297 + https://github.com/cs3org/reva/pull/2332 + https://github.com/cs3org/reva/pull/2346 + + * Bugfix #2316: Fix the share types in propfinds + + The share types for public links were not correctly added to propfinds. + + https://github.com/cs3org/reva/pull/2316 + + * Bugfix #2803: Fix app provider for editor public links + + Fixed opening the app provider in public links with the editor permission. The app provider + failed to open the file in read write mode. + + https://github.com/owncloud/ocis/issues/2803 + https://github.com/cs3org/reva/pull/2310 + + * Bugfix #2298: Remove share refs from trashbin + + https://github.com/cs3org/reva/pull/2298 + + * Bugfix #2309: Remove early finish for zero byte file uploads + + We've fixed the upload of zero byte files by removing the early upload finishing mechanism. + + https://github.com/cs3org/reva/issues/2309 + https://github.com/owncloud/ocis/issues/2609 + + * Bugfix #1941: Fix TUS uploads with transfer token only + + TUS uploads had been stopped when the user JWT token expired, even if only the transfer token + should be validated. Now uploads will continue as intended. + + https://github.com/cs3org/reva/pull/1941 + + * Change #2210: Fix app provider new file creation and improved error codes + + We've fixed the behavior for the app provider when creating new files. Previously the app + provider would overwrite already existing files when creating a new file, this is now handled + and prevented. The new file endpoint accepted a path to a file, but this does not work for spaces. + Therefore we now use the resource id of the folder where the file should be created and a filename + to create the new file. Also the app provider returns more useful error codes in a lot of cases. + + https://github.com/cs3org/reva/pull/2210 + + * Enhancement #2217: OIDC auth driver for ESCAPE IAM + + This enhancement allows for oidc token authentication via the ESCAPE IAM service. + Authentication relies on mappings of ESCAPE IAM groups to REVA users. For a valid token, if at + the most one group from the groups claim is mapped to one REVA user, authentication can take + place. + + https://github.com/cs3org/reva/pull/2217 + + * Enhancement #2256: Return user type in the response of the ocs GET user call + + https://github.com/cs3org/reva/pull/2256 + + * Enhancement #2315: Add new attributes to public link propfinds + + Added a new property "oc:signature-auth" to public link propfinds. This is a necessary change + to be able to support archive downloads in password protected public links. + + https://github.com/cs3org/reva/pull/2315 + + * Enhancement #2740: Implement space membership endpoints + + Implemented endpoints to add and remove members to spaces. + + https://github.com/owncloud/ocis/issues/2740 + https://github.com/cs3org/reva/pull/2250 + + * Enhancement #2252: Add the xattr sys.acl to SysACL (eosgrpc) + + https://github.com/cs3org/reva/pull/2252 + + * Enhancement #2314: OIDC: fallback if IDP doesn't provide "preferred_username" claim + + Some IDPs don't support the "preferred_username" claim. Fallback to the "email" claim in that + case. + + https://github.com/cs3org/reva/pull/2314 + + From 778de37266ff27af25023538ca8aa28fe68a0150 Mon Sep 17 00:00:00 2001 From: Michael Barz Date: Sun, 12 Dec 2021 16:12:13 +0100 Subject: [PATCH 13/58] handle non existent spaces gracefully (#2354) --- changelog/unreleased/update-handle-empty-spaces.md | 5 +++++ pkg/storage/utils/decomposedfs/spaces.go | 13 ++++++++++++- 2 files changed, 17 insertions(+), 1 deletion(-) create mode 100644 changelog/unreleased/update-handle-empty-spaces.md diff --git a/changelog/unreleased/update-handle-empty-spaces.md b/changelog/unreleased/update-handle-empty-spaces.md new file mode 100644 index 0000000000..6ee2ebe634 --- /dev/null +++ b/changelog/unreleased/update-handle-empty-spaces.md @@ -0,0 +1,5 @@ +Change: Return not found when updating non existent space + +If a spaceid of a space which is updated doesn't exist, handle it as a not found error. + +https://github.com/cs3org/reva/pull/2354 diff --git a/pkg/storage/utils/decomposedfs/spaces.go b/pkg/storage/utils/decomposedfs/spaces.go index 8c61ecf1e1..b34bb57f65 100644 --- a/pkg/storage/utils/decomposedfs/spaces.go +++ b/pkg/storage/utils/decomposedfs/spaces.go @@ -254,7 +254,12 @@ func (fs *Decomposedfs) UpdateStorageSpace(ctx context.Context, req *provider.Up } if len(matches) != 1 { - return nil, fmt.Errorf("update space failed: found %d matching spaces", len(matches)) + return &provider.UpdateStorageSpaceResponse{ + Status: &v1beta11.Status{ + Code: v1beta11.Code_CODE_NOT_FOUND, + Message: fmt.Sprintf("update space failed: found %d matching spaces", len(matches)), + }, + }, nil } target, err := os.Readlink(matches[0]) @@ -267,6 +272,12 @@ func (fs *Decomposedfs) UpdateStorageSpace(ctx context.Context, req *provider.Up return nil, err } + u, ok := ctxpkg.ContextGetUser(ctx) + if !ok { + return nil, fmt.Errorf("decomposedfs: spaces: contextual user not found") + } + space.Owner = u + if space.Name != "" { if err := node.SetMetadata(xattrs.SpaceNameAttr, space.Name); err != nil { return nil, err From 7e9eff358991d9bb6e63418c940cfa5b0fe90355 Mon Sep 17 00:00:00 2001 From: Swikriti Tripathi <41103328+SwikritiT@users.noreply.github.com> Date: Mon, 13 Dec 2021 21:30:18 +0545 Subject: [PATCH 14/58] Bump core commit id for tests (#2365) --- .drone.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.env b/.drone.env index cce5230458..c8ba576897 100644 --- a/.drone.env +++ b/.drone.env @@ -1,3 +1,3 @@ # The test runner source for API tests -CORE_COMMITID=4bbd91de68aefdf94c03d2920a3353702f856280 +CORE_COMMITID=5f98837bd95d75660fc9fbc9a8802f7db8793a89 CORE_BRANCH=acceptance-test-changes-waiting-2021-11 From 30b629ed978d9f5252c5181d0b9e85cd2fb55f27 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Dec 2021 21:15:42 +0530 Subject: [PATCH 15/58] [Build-deps]: Bump github.com/minio/minio-go/v7 from 7.0.16 to 7.0.18 (#2363) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 167002151e..2f0bdf11aa 100644 --- a/go.mod +++ b/go.mod @@ -41,7 +41,7 @@ require ( github.com/juliangruber/go-intersect v1.1.0 github.com/mattn/go-sqlite3 v1.14.9 github.com/mileusna/useragent v1.0.2 - github.com/minio/minio-go/v7 v7.0.16 + github.com/minio/minio-go/v7 v7.0.18 github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/go-testing-interface v1.14.1 // indirect github.com/mitchellh/mapstructure v1.4.3 diff --git a/go.sum b/go.sum index dd5a0265b6..6ee774d844 100644 --- a/go.sum +++ b/go.sum @@ -468,8 +468,8 @@ github.com/mileusna/useragent v1.0.2 h1:DgVKtiPnjxlb73z9bCwgdUvU2nQNQ97uhgfO8l9u github.com/mileusna/useragent v1.0.2/go.mod h1:3d8TOmwL/5I8pJjyVDteHtgDGcefrFUX4ccGOMKNYYc= github.com/minio/md5-simd v1.1.0 h1:QPfiOqlZH+Cj9teu0t9b1nTBfPbyTl16Of5MeuShdK4= github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw= -github.com/minio/minio-go/v7 v7.0.16 h1:GspaSBS8lOuEUCAqMe0W3UxSoyOA4b4F8PTspRVI+k4= -github.com/minio/minio-go/v7 v7.0.16/go.mod h1:pUV0Pc+hPd1nccgmzQF/EXh48l/Z/yps6QPF1aaie4g= +github.com/minio/minio-go/v7 v7.0.18 h1:fncn6iacnK+i2uYfNc5aVPG7bEqQH0nU4yAGMSunY0w= +github.com/minio/minio-go/v7 v7.0.18/go.mod h1:SyQ1IFeJuaa+eV5yEDxW7hYE1s5VVq5sgImDe27R+zg= github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU= github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= From 2fae455725c1652349748372421882d49bb6d1db Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Dec 2021 21:17:23 +0530 Subject: [PATCH 16/58] [Build-deps]: Bump github.com/ReneKroon/ttlcache/v2 from 2.9.0 to 2.10.0 (#2358) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 2f0bdf11aa..086baed2fe 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/Masterminds/goutils v1.1.1 // indirect github.com/Masterminds/semver v1.5.0 // indirect github.com/Masterminds/sprig v2.22.0+incompatible - github.com/ReneKroon/ttlcache/v2 v2.9.0 + github.com/ReneKroon/ttlcache/v2 v2.10.0 github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect github.com/aws/aws-sdk-go v1.42.19 github.com/beevik/etree v1.1.0 diff --git a/go.sum b/go.sum index 6ee774d844..07b1c13c72 100644 --- a/go.sum +++ b/go.sum @@ -52,8 +52,8 @@ github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAE github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= -github.com/ReneKroon/ttlcache/v2 v2.9.0 h1:NzwfErbifoNA3djEGwQJXKp/386imbyrc6Qmns5IX7c= -github.com/ReneKroon/ttlcache/v2 v2.9.0/go.mod h1:mBxvsNY+BT8qLLd6CuAJubbKo6r0jh3nb5et22bbfGY= +github.com/ReneKroon/ttlcache/v2 v2.10.0 h1:y4g2gs2fSqAugSseq7vUB1jClqzT/e0CjSrMJywoWa8= +github.com/ReneKroon/ttlcache/v2 v2.10.0/go.mod h1:mBxvsNY+BT8qLLd6CuAJubbKo6r0jh3nb5et22bbfGY= github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= From 0771dfa7cfd52585b6d6e67254c37bdcf9fdeaec Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Dec 2021 21:17:45 +0530 Subject: [PATCH 17/58] [Build-deps]: Bump go.opentelemetry.io/otel/exporters/jaeger (#2362) --- go.mod | 8 ++++---- go.sum | 19 +++++++++++++------ 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/go.mod b/go.mod index 086baed2fe..168ad5d487 100644 --- a/go.mod +++ b/go.mod @@ -65,10 +65,10 @@ require ( go.mongodb.org/mongo-driver v1.7.2 // indirect go.opencensus.io v0.23.0 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.27.0 - go.opentelemetry.io/otel v1.2.0 - go.opentelemetry.io/otel/exporters/jaeger v1.2.0 - go.opentelemetry.io/otel/sdk v1.2.0 - go.opentelemetry.io/otel/trace v1.2.0 + go.opentelemetry.io/otel v1.3.0 + go.opentelemetry.io/otel/exporters/jaeger v1.3.0 + go.opentelemetry.io/otel/sdk v1.3.0 + go.opentelemetry.io/otel/trace v1.3.0 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f golang.org/x/sys v0.0.0-20210921065528-437939a70204 diff --git a/go.sum b/go.sum index 07b1c13c72..11ef1e3104 100644 --- a/go.sum +++ b/go.sum @@ -153,6 +153,11 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0 h1:TrB8swr/68K7m9CcGut2g3UOihhbcbiMAYiuTXdEih4= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.1 h1:DX7uPQ4WgAWfoh+NGGlbJQswnYIVvz0SRlLS3rPZQDA= +github.com/go-logr/logr v1.2.1/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/stdr v1.2.0 h1:j4LrlVXgrbIWO83mmQUnK0Hi+YnbD+vzrE1z/EphbFE= +github.com/go-logr/stdr v1.2.0/go.mod h1:YkVgnZu1ZjjL7xTxrfm/LLZBfkhTqSR1ydtm6jTKKwI= github.com/go-openapi/analysis v0.0.0-20180825180245-b006789cd277/go.mod h1:k70tL6pCuVxPJOHXQ+wIac1FUrvNkHolPie/cLEU6hI= github.com/go-openapi/analysis v0.17.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= github.com/go-openapi/analysis v0.18.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= @@ -666,14 +671,16 @@ go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.27.0 h1:TON1iU3Y5oIytGQHIejDYLam5uoSMsmA0UV9Yupb5gQ= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.27.0/go.mod h1:T/zQwBldOpoAEpE3HMbLnI8ydESZVz4ggw6Is4FF9LI= -go.opentelemetry.io/otel v1.2.0 h1:YOQDvxO1FayUcT9MIhJhgMyNO1WqoduiyvQHzGN0kUQ= go.opentelemetry.io/otel v1.2.0/go.mod h1:aT17Fk0Z1Nor9e0uisf98LrntPGMnk4frBO9+dkf69I= -go.opentelemetry.io/otel/exporters/jaeger v1.2.0 h1:C/5Egj3MJBXRJi22cSl07suqPqtZLnLFmH//OxETUEc= -go.opentelemetry.io/otel/exporters/jaeger v1.2.0/go.mod h1:KJLFbEMKTNPIfOxcg/WikIozEoKcPgJRz3Ce1vLlM8E= -go.opentelemetry.io/otel/sdk v1.2.0 h1:wKN260u4DesJYhyjxDa7LRFkuhH7ncEVKU37LWcyNIo= -go.opentelemetry.io/otel/sdk v1.2.0/go.mod h1:jNN8QtpvbsKhgaC6V5lHiejMoKD+V8uadoSafgHPx1U= -go.opentelemetry.io/otel/trace v1.2.0 h1:Ys3iqbqZhcf28hHzrm5WAquMkDHNZTUkw7KHbuNjej0= +go.opentelemetry.io/otel v1.3.0 h1:APxLf0eiBwLl+SOXiJJCVYzA1OOJNyAoV8C5RNRyy7Y= +go.opentelemetry.io/otel v1.3.0/go.mod h1:PWIKzi6JCp7sM0k9yZ43VX+T345uNbAkDKwHVjb2PTs= +go.opentelemetry.io/otel/exporters/jaeger v1.3.0 h1:HfydzioALdtcB26H5WHc4K47iTETJCdloL7VN579/L0= +go.opentelemetry.io/otel/exporters/jaeger v1.3.0/go.mod h1:KoYHi1BtkUPncGSRtCe/eh1ijsnePhSkxwzz07vU0Fc= +go.opentelemetry.io/otel/sdk v1.3.0 h1:3278edCoH89MEJ0Ky8WQXVmDQv3FX4ZJ3Pp+9fJreAI= +go.opentelemetry.io/otel/sdk v1.3.0/go.mod h1:rIo4suHNhQwBIPg9axF8V9CA72Wz2mKF1teNrup8yzs= go.opentelemetry.io/otel/trace v1.2.0/go.mod h1:N5FLswTubnxKxOJHM7XZC074qpeEdLy3CgAVsdMucK0= +go.opentelemetry.io/otel/trace v1.3.0 h1:doy8Hzb1RJ+I3yFhtDmwNc7tIyw1tNMOIsyPzp1NOGY= +go.opentelemetry.io/otel/trace v1.3.0/go.mod h1:c/VDhno8888bvQYmbYLqe41/Ldmr/KKunbvWM4/fEjk= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.1.10 h1:z+mqJhf6ss6BSfSM671tgKyZBFPTTJM+HLxnhPC3wu0= From 8236ffa3f824cd7129e086b00c7329c36905d467 Mon Sep 17 00:00:00 2001 From: Willy Kloucek <34452982+wkloucek@users.noreply.github.com> Date: Wed, 15 Dec 2021 08:45:11 +0100 Subject: [PATCH 18/58] fix tests by pointing to the right owncloud/core commit id for tests (#2375) --- .drone.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.env b/.drone.env index c8ba576897..71e1a71ad8 100644 --- a/.drone.env +++ b/.drone.env @@ -1,3 +1,3 @@ # The test runner source for API tests -CORE_COMMITID=5f98837bd95d75660fc9fbc9a8802f7db8793a89 +CORE_COMMITID=2987544bfd6f6823df68f6ea392ba41c1ea4f07c CORE_BRANCH=acceptance-test-changes-waiting-2021-11 From f76d428f4bed52eba909cf0a3787f603337354a7 Mon Sep 17 00:00:00 2001 From: Willy Kloucek <34452982+wkloucek@users.noreply.github.com> Date: Wed, 15 Dec 2021 14:27:23 +0100 Subject: [PATCH 19/58] add new file capabilties to ocs for the app provider (#2379) --- .../unreleased/enhancement-add-new-file-capability.md | 8 ++++++++ internal/http/services/owncloud/ocs/data/capabilities.go | 1 + 2 files changed, 9 insertions(+) create mode 100644 changelog/unreleased/enhancement-add-new-file-capability.md diff --git a/changelog/unreleased/enhancement-add-new-file-capability.md b/changelog/unreleased/enhancement-add-new-file-capability.md new file mode 100644 index 0000000000..23805aeff3 --- /dev/null +++ b/changelog/unreleased/enhancement-add-new-file-capability.md @@ -0,0 +1,8 @@ +Enhancement: add new file url of the app provider to the ocs capabilities + +We've added the new file capability of the app provider to the ocs capabilities, so that +clients can discover this url analogous to the app list and file open urls. + +https://github.com/cs3org/reva/pull/2379 +https://github.com/owncloud/ocis/pull/2884 +https://github.com/owncloud/web/pull/5890#issuecomment-993905242 diff --git a/internal/http/services/owncloud/ocs/data/capabilities.go b/internal/http/services/owncloud/ocs/data/capabilities.go index 3c3c693952..81ee4e3320 100644 --- a/internal/http/services/owncloud/ocs/data/capabilities.go +++ b/internal/http/services/owncloud/ocs/data/capabilities.go @@ -117,6 +117,7 @@ type CapabilitiesAppProvider struct { Version string `json:"version" xml:"version" mapstructure:"version"` AppsURL string `json:"apps_url" xml:"apps_url" mapstructure:"apps_url"` OpenURL string `json:"open_url" xml:"open_url" mapstructure:"open_url"` + NewURL string `json:"new_url" xml:"new_url" mapstructure:"new_url"` } // CapabilitiesFiles TODO this is storage specific, not global. What effect do these options have on the clients? From 8c6bad69fc49a83dba356fc30f6703b561da9bd4 Mon Sep 17 00:00:00 2001 From: Swikriti Tripathi <41103328+SwikritiT@users.noreply.github.com> Date: Wed, 15 Dec 2021 19:12:54 +0545 Subject: [PATCH 20/58] Remove test from expected to fail and bump commit id (#2380) --- .drone.env | 2 +- tests/acceptance/expected-failures-on-OCIS-storage.md | 3 --- tests/acceptance/expected-failures-on-S3NG-storage.md | 3 --- 3 files changed, 1 insertion(+), 7 deletions(-) diff --git a/.drone.env b/.drone.env index 71e1a71ad8..b07b2f80cc 100644 --- a/.drone.env +++ b/.drone.env @@ -1,3 +1,3 @@ # The test runner source for API tests -CORE_COMMITID=2987544bfd6f6823df68f6ea392ba41c1ea4f07c +CORE_COMMITID=38ee68e6358443e980ba5e7036cf1fb554443814 CORE_BRANCH=acceptance-test-changes-waiting-2021-11 diff --git a/tests/acceptance/expected-failures-on-OCIS-storage.md b/tests/acceptance/expected-failures-on-OCIS-storage.md index 100f8ae9f8..a509283234 100644 --- a/tests/acceptance/expected-failures-on-OCIS-storage.md +++ b/tests/acceptance/expected-failures-on-OCIS-storage.md @@ -1365,6 +1365,3 @@ _ocs: api compatibility, return correct status code_ ### [Content-type is not multipart/byteranges when downloading file with Range Header](https://github.com/owncloud/ocis/issues/2677) - [apiWebdavOperations/downloadFile.feature:169](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavOperations/downloadFile.feature#L169) - [apiWebdavOperations/downloadFile.feature:170](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavOperations/downloadFile.feature#L170) - -### [send PUT requests to another user's webDav endpoints as normal user](https://github.com/owncloud/ocis/issues/2759) -- [apiAuthWebDav/webDavPUTAuth.feature:40](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiAuthWebDav/webDavPUTAuth.feature#L40) diff --git a/tests/acceptance/expected-failures-on-S3NG-storage.md b/tests/acceptance/expected-failures-on-S3NG-storage.md index 7fc64d6809..9c7bfff96b 100644 --- a/tests/acceptance/expected-failures-on-S3NG-storage.md +++ b/tests/acceptance/expected-failures-on-S3NG-storage.md @@ -1365,6 +1365,3 @@ _ocs: api compatibility, return correct status code_ ### [Content-type is not multipart/byteranges when downloading file with Range Header](https://github.com/owncloud/ocis/issues/2677) - [apiWebdavOperations/downloadFile.feature:169](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavOperations/downloadFile.feature#L169) - [apiWebdavOperations/downloadFile.feature:170](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavOperations/downloadFile.feature#L170) - -### [send PUT requests to another user's webDav endpoints as normal user](https://github.com/owncloud/ocis/issues/2759) -- [apiAuthWebDav/webDavPUTAuth.feature:40](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiAuthWebDav/webDavPUTAuth.feature#L40) From e65efdcb40a0e5c6e12c2943d6e4044e96a875b0 Mon Sep 17 00:00:00 2001 From: Phil Davis Date: Wed, 15 Dec 2021 19:13:23 +0545 Subject: [PATCH 21/58] add .drone.env to CODEOWNERS as it is part of the test files (#2378) --- CODEOWNERS | 1 + 1 file changed, 1 insertion(+) diff --git a/CODEOWNERS b/CODEOWNERS index 4c9822be14..f22f6c60db 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -1,4 +1,5 @@ * @labkode @ishank011 +.drone.env @labkode @ishank011 @glpatcern @cs3org/owncloud-team changelog @labkode @ishank011 @glpatcern @cs3org/owncloud-team examples @labkode @ishank011 @glpatcern @cs3org/owncloud-team tests @labkode @ishank011 @glpatcern @cs3org/owncloud-team From 5cde0187454be9e5de90cca00eedca2473ee82db Mon Sep 17 00:00:00 2001 From: Willy Kloucek <34452982+wkloucek@users.noreply.github.com> Date: Wed, 15 Dec 2021 14:29:08 +0100 Subject: [PATCH 22/58] fix webdav copy for zero byte files (#2374) --- .../fix-webdav-copy-zero-byte-file.md | 9 ++++ internal/http/services/owncloud/ocdav/copy.go | 53 +++++++++---------- 2 files changed, 34 insertions(+), 28 deletions(-) create mode 100644 changelog/unreleased/fix-webdav-copy-zero-byte-file.md diff --git a/changelog/unreleased/fix-webdav-copy-zero-byte-file.md b/changelog/unreleased/fix-webdav-copy-zero-byte-file.md new file mode 100644 index 0000000000..2fc971cd54 --- /dev/null +++ b/changelog/unreleased/fix-webdav-copy-zero-byte-file.md @@ -0,0 +1,9 @@ +Bugfix: Fix webdav copy of zero byte files + +We've fixed the webdav copy action of zero byte files, which was not performed +because the webdav api assumed, that zero byte uploads are created when initiating +the upload, which was recently removed from all storage drivers. Therefore the +webdav api also uploads zero byte files after initiating the upload. + +https://github.com/cs3org/reva/pull/2374 +https://github.com/cs3org/reva/pull/2309 diff --git a/internal/http/services/owncloud/ocdav/copy.go b/internal/http/services/owncloud/ocdav/copy.go index 8663d48c6a..eefb8d74e1 100644 --- a/internal/http/services/owncloud/ocdav/copy.go +++ b/internal/http/services/owncloud/ocdav/copy.go @@ -237,22 +237,21 @@ func (s *svc) executePathCopy(ctx context.Context, client gateway.GatewayAPIClie // 4. do upload - if cp.sourceInfo.GetSize() > 0 { - httpUploadReq, err := rhttp.NewRequest(ctx, "PUT", uploadEP, httpDownloadRes.Body) - if err != nil { - return err - } - httpUploadReq.Header.Set(datagateway.TokenTransportHeader, uploadToken) + httpUploadReq, err := rhttp.NewRequest(ctx, "PUT", uploadEP, httpDownloadRes.Body) + if err != nil { + return err + } + httpUploadReq.Header.Set(datagateway.TokenTransportHeader, uploadToken) - httpUploadRes, err := s.client.Do(httpUploadReq) - if err != nil { - return err - } - defer httpUploadRes.Body.Close() - if httpUploadRes.StatusCode != http.StatusOK { - return err - } + httpUploadRes, err := s.client.Do(httpUploadReq) + if err != nil { + return err + } + defer httpUploadRes.Body.Close() + if httpUploadRes.StatusCode != http.StatusOK { + return err } + } return nil } @@ -457,21 +456,19 @@ func (s *svc) executeSpacesCopy(ctx context.Context, w http.ResponseWriter, clie // 4. do upload - if cp.sourceInfo.GetSize() > 0 { - httpUploadReq, err := rhttp.NewRequest(ctx, http.MethodPut, uploadEP, httpDownloadRes.Body) - if err != nil { - return err - } - httpUploadReq.Header.Set(datagateway.TokenTransportHeader, uploadToken) + httpUploadReq, err := rhttp.NewRequest(ctx, http.MethodPut, uploadEP, httpDownloadRes.Body) + if err != nil { + return err + } + httpUploadReq.Header.Set(datagateway.TokenTransportHeader, uploadToken) - httpUploadRes, err := s.client.Do(httpUploadReq) - if err != nil { - return err - } - defer httpUploadRes.Body.Close() - if httpUploadRes.StatusCode != http.StatusOK { - return err - } + httpUploadRes, err := s.client.Do(httpUploadReq) + if err != nil { + return err + } + defer httpUploadRes.Body.Close() + if httpUploadRes.StatusCode != http.StatusOK { + return err } } return nil From f3c8b621731124c49061c4375566e259809dee02 Mon Sep 17 00:00:00 2001 From: Willy Kloucek <34452982+wkloucek@users.noreply.github.com> Date: Fri, 17 Dec 2021 12:02:16 +0100 Subject: [PATCH 23/58] Implement touch file (#2369) * implement cs3org/cs3apis#154 * use TouchFile for the app provider * add changelog and comments * revert use TouchFile in app provider * fix resource typo Co-authored-by: Giuseppe Lo Presti Co-authored-by: Giuseppe Lo Presti --- .../unreleased/enhancement-add-touch-file.md | 6 ++++ go.mod | 2 +- go.sum | 4 +++ internal/grpc/interceptors/auth/scope.go | 2 ++ .../grpc/interceptors/readonly/readonly.go | 6 +++- .../grpc/services/gateway/storageprovider.go | 19 ++++++++++++ .../publicstorageprovider.go | 13 ++++++++ .../storageprovider/storageprovider.go | 30 +++++++++++++++++++ pkg/auth/scope/publicshare.go | 2 ++ pkg/auth/scope/resourceinfo.go | 2 ++ pkg/auth/scope/share.go | 2 ++ pkg/storage/fs/nextcloud/nextcloud.go | 5 ++++ pkg/storage/fs/owncloud/owncloud.go | 5 ++++ pkg/storage/fs/owncloudsql/owncloudsql.go | 5 ++++ pkg/storage/fs/s3/s3.go | 5 ++++ pkg/storage/storage.go | 1 + .../utils/decomposedfs/decomposedfs.go | 6 ++++ pkg/storage/utils/eosfs/eosfs.go | 23 ++++++++++++++ pkg/storage/utils/localfs/localfs.go | 5 ++++ 19 files changed, 141 insertions(+), 2 deletions(-) create mode 100644 changelog/unreleased/enhancement-add-touch-file.md diff --git a/changelog/unreleased/enhancement-add-touch-file.md b/changelog/unreleased/enhancement-add-touch-file.md new file mode 100644 index 0000000000..8e9c04e0b9 --- /dev/null +++ b/changelog/unreleased/enhancement-add-touch-file.md @@ -0,0 +1,6 @@ +Enhancement: Implement TouchFile from the CS3apis + +We've updated the CS3apis and implemented the TouchFile method. + +https://github.com/cs3org/reva/pull/2369 +https://github.com/cs3org/cs3apis/pull/154 diff --git a/go.mod b/go.mod index 168ad5d487..b5b9d08395 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/cheggaaa/pb v1.0.29 github.com/coreos/go-oidc v2.2.1+incompatible github.com/cs3org/cato v0.0.0-20200828125504-e418fc54dd5e - github.com/cs3org/go-cs3apis v0.0.0-20211104090126-8e972dca8304 + github.com/cs3org/go-cs3apis v0.0.0-20211213090556-12c0d565f51d github.com/cubewise-code/go-mime v0.0.0-20200519001935-8c5762b177d8 github.com/eventials/go-tus v0.0.0-20200718001131-45c7ec8f5d59 github.com/gdexlab/go-render v1.0.1 diff --git a/go.sum b/go.sum index 11ef1e3104..c937822ad9 100644 --- a/go.sum +++ b/go.sum @@ -111,8 +111,12 @@ github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSV github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cs3org/cato v0.0.0-20200828125504-e418fc54dd5e h1:tqSPWQeueWTKnJVMJffz4pz0o1WuQxJ28+5x5JgaHD8= github.com/cs3org/cato v0.0.0-20200828125504-e418fc54dd5e/go.mod h1:XJEZ3/EQuI3BXTp/6DUzFr850vlxq11I6satRtz0YQ4= +github.com/cs3org/go-cs3apis v0.0.0-20210325133324-32b03d75a535 h1:555D8A3ddKqb4OyK9v5mdphw2zDLWKGXOkcnf1RQwTA= +github.com/cs3org/go-cs3apis v0.0.0-20210325133324-32b03d75a535/go.mod h1:UXha4TguuB52H14EMoSsCqDj7k8a/t7g4gVP+bgY5LY= github.com/cs3org/go-cs3apis v0.0.0-20211104090126-8e972dca8304 h1:e/nIPR518vyvrulo9goAZTtYD6gFfu/2/9MDe6mTGcw= github.com/cs3org/go-cs3apis v0.0.0-20211104090126-8e972dca8304/go.mod h1:UXha4TguuB52H14EMoSsCqDj7k8a/t7g4gVP+bgY5LY= +github.com/cs3org/go-cs3apis v0.0.0-20211213090556-12c0d565f51d h1:gnb2ciU4N+RwUug/nwe54wenWi7vSp5bAAjXINlgHZ8= +github.com/cs3org/go-cs3apis v0.0.0-20211213090556-12c0d565f51d/go.mod h1:UXha4TguuB52H14EMoSsCqDj7k8a/t7g4gVP+bgY5LY= github.com/cubewise-code/go-mime v0.0.0-20200519001935-8c5762b177d8 h1:Z9lwXumT5ACSmJ7WGnFl+OMLLjpz5uR2fyz7dC255FI= github.com/cubewise-code/go-mime v0.0.0-20200519001935-8c5762b177d8/go.mod h1:4abs/jPXcmJzYoYGF91JF9Uq9s/KL5n1jvFDix8KcqY= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= diff --git a/internal/grpc/interceptors/auth/scope.go b/internal/grpc/interceptors/auth/scope.go index 8c9a907ce3..8bad148478 100644 --- a/internal/grpc/interceptors/auth/scope.go +++ b/internal/grpc/interceptors/auth/scope.go @@ -241,6 +241,8 @@ func extractRef(req interface{}, hasEditorRole bool) (*provider.Reference, bool) // Write Requests case *provider.CreateContainerRequest: return v.GetRef(), true + case *provider.TouchFileRequest: + return v.GetRef(), true case *provider.DeleteRequest: return v.GetRef(), true case *provider.MoveRequest: diff --git a/internal/grpc/interceptors/readonly/readonly.go b/internal/grpc/interceptors/readonly/readonly.go index f82a7ab46b..d1d8801c2d 100644 --- a/internal/grpc/interceptors/readonly/readonly.go +++ b/internal/grpc/interceptors/readonly/readonly.go @@ -99,7 +99,11 @@ func NewUnary(map[string]interface{}) (grpc.UnaryServerInterceptor, int, error) }, nil case *provider.CreateContainerRequest: return &provider.CreateContainerResponse{ - Status: rstatus.NewPermissionDenied(ctx, nil, "permission denied: tried to create resoure on readonly storage"), + Status: rstatus.NewPermissionDenied(ctx, nil, "permission denied: tried to create resource on read-only storage"), + }, nil + case *provider.TouchFileRequest: + return &provider.TouchFileResponse{ + Status: rstatus.NewPermissionDenied(ctx, nil, "permission denied: tried to create resource on read-only storage"), }, nil case *provider.CreateHomeRequest: return &provider.CreateHomeResponse{ diff --git a/internal/grpc/services/gateway/storageprovider.go b/internal/grpc/services/gateway/storageprovider.go index 66599a324e..cdd622e9c7 100644 --- a/internal/grpc/services/gateway/storageprovider.go +++ b/internal/grpc/services/gateway/storageprovider.go @@ -851,6 +851,25 @@ func (s *svc) createContainer(ctx context.Context, req *provider.CreateContainer return res, nil } +func (s *svc) TouchFile(ctx context.Context, req *provider.TouchFileRequest) (*provider.TouchFileResponse, error) { + c, err := s.find(ctx, req.Ref) + if err != nil { + return &provider.TouchFileResponse{ + Status: status.NewStatusFromErrType(ctx, "TouchFile ref="+req.Ref.String(), err), + }, nil + } + + res, err := c.TouchFile(ctx, req) + if err != nil { + if gstatus.Code(err) == codes.PermissionDenied { + return &provider.TouchFileResponse{Status: &rpc.Status{Code: rpc.Code_CODE_PERMISSION_DENIED}}, nil + } + return nil, errors.Wrap(err, "gateway: error calling TouchFile") + } + + return res, nil +} + // check if the path contains the prefix of the shared folder func (s *svc) inSharedFolder(ctx context.Context, p string) bool { sharedFolder := s.getSharedFolder(ctx) diff --git a/internal/grpc/services/publicstorageprovider/publicstorageprovider.go b/internal/grpc/services/publicstorageprovider/publicstorageprovider.go index c6ec4e983e..80b64f4e99 100644 --- a/internal/grpc/services/publicstorageprovider/publicstorageprovider.go +++ b/internal/grpc/services/publicstorageprovider/publicstorageprovider.go @@ -363,6 +363,19 @@ func (s *service) CreateContainer(ctx context.Context, req *provider.CreateConta return res, nil } +func (s *service) TouchFile(ctx context.Context, req *provider.TouchFileRequest) (*provider.TouchFileResponse, error) { + ref, _, _, st, err := s.translatePublicRefToCS3Ref(ctx, req.Ref) + switch { + case err != nil: + return nil, err + case st != nil: + return &provider.TouchFileResponse{ + Status: st, + }, nil + } + return s.gateway.TouchFile(ctx, &provider.TouchFileRequest{Opaque: req.Opaque, Ref: ref}) +} + func (s *service) Delete(ctx context.Context, req *provider.DeleteRequest) (*provider.DeleteResponse, error) { ctx, span := rtrace.Provider.Tracer("publicstorageprovider").Start(ctx, "Delete") defer span.End() diff --git a/internal/grpc/services/storageprovider/storageprovider.go b/internal/grpc/services/storageprovider/storageprovider.go index aac7f68453..a0d659005a 100644 --- a/internal/grpc/services/storageprovider/storageprovider.go +++ b/internal/grpc/services/storageprovider/storageprovider.go @@ -544,6 +544,36 @@ func (s *service) CreateContainer(ctx context.Context, req *provider.CreateConta return res, nil } +func (s *service) TouchFile(ctx context.Context, req *provider.TouchFileRequest) (*provider.TouchFileResponse, error) { + newRef, err := s.unwrap(ctx, req.Ref) + if err != nil { + return &provider.TouchFileResponse{ + Status: status.NewInternal(ctx, err, "error unwrapping path"), + }, nil + } + if err := s.storage.TouchFile(ctx, newRef); err != nil { + var st *rpc.Status + switch err.(type) { + case errtypes.IsNotFound: + st = status.NewNotFound(ctx, "path not found when touching the file") + case errtypes.AlreadyExists: + st = status.NewAlreadyExists(ctx, err, "file already exists") + case errtypes.PermissionDenied: + st = status.NewPermissionDenied(ctx, err, "permission denied") + default: + st = status.NewInternal(ctx, err, "error touching file: "+req.Ref.String()) + } + return &provider.TouchFileResponse{ + Status: st, + }, nil + } + + res := &provider.TouchFileResponse{ + Status: status.NewOK(ctx), + } + return res, nil +} + func (s *service) Delete(ctx context.Context, req *provider.DeleteRequest) (*provider.DeleteResponse, error) { newRef, err := s.unwrap(ctx, req.Ref) if err != nil { diff --git a/pkg/auth/scope/publicshare.go b/pkg/auth/scope/publicshare.go index b7f7d078b6..7de007ca87 100644 --- a/pkg/auth/scope/publicshare.go +++ b/pkg/auth/scope/publicshare.go @@ -57,6 +57,8 @@ func publicshareScope(ctx context.Context, scope *authpb.Scope, resource interfa // need to return appropriate status codes in the ocs/ocdav layers. case *provider.CreateContainerRequest: return hasRoleEditor(*scope) && checkStorageRef(ctx, &share, v.GetRef()), nil + case *provider.TouchFileRequest: + return hasRoleEditor(*scope) && checkStorageRef(ctx, &share, v.GetRef()), nil case *provider.DeleteRequest: return hasRoleEditor(*scope) && checkStorageRef(ctx, &share, v.GetRef()), nil case *provider.MoveRequest: diff --git a/pkg/auth/scope/resourceinfo.go b/pkg/auth/scope/resourceinfo.go index f5afe94cfe..1a6c41a40f 100644 --- a/pkg/auth/scope/resourceinfo.go +++ b/pkg/auth/scope/resourceinfo.go @@ -55,6 +55,8 @@ func resourceinfoScope(_ context.Context, scope *authpb.Scope, resource interfac // need to return appropriate status codes in the ocs/ocdav layers. case *provider.CreateContainerRequest: return hasRoleEditor(*scope) && checkResourceInfo(&r, v.GetRef()), nil + case *provider.TouchFileRequest: + return hasRoleEditor(*scope) && checkResourceInfo(&r, v.GetRef()), nil case *provider.DeleteRequest: return hasRoleEditor(*scope) && checkResourceInfo(&r, v.GetRef()), nil case *provider.MoveRequest: diff --git a/pkg/auth/scope/share.go b/pkg/auth/scope/share.go index 1e29fd44e1..48683dea17 100644 --- a/pkg/auth/scope/share.go +++ b/pkg/auth/scope/share.go @@ -56,6 +56,8 @@ func shareScope(_ context.Context, scope *authpb.Scope, resource interface{}, lo // need to return appropriate status codes in the ocs/ocdav layers. case *provider.CreateContainerRequest: return checkShareStorageRef(&share, v.GetRef()), nil + case *provider.TouchFileRequest: + return checkShareStorageRef(&share, v.GetRef()), nil case *provider.DeleteRequest: return checkShareStorageRef(&share, v.GetRef()), nil case *provider.MoveRequest: diff --git a/pkg/storage/fs/nextcloud/nextcloud.go b/pkg/storage/fs/nextcloud/nextcloud.go index 7ee908beda..567a880863 100644 --- a/pkg/storage/fs/nextcloud/nextcloud.go +++ b/pkg/storage/fs/nextcloud/nextcloud.go @@ -253,6 +253,11 @@ func (nc *StorageDriver) CreateDir(ctx context.Context, ref *provider.Reference) return err } +// TouchFile as defined in the storage.FS interface +func (nc *StorageDriver) TouchFile(ctx context.Context, ref *provider.Reference) error { + return fmt.Errorf("unimplemented: TouchFile") +} + // Delete as defined in the storage.FS interface func (nc *StorageDriver) Delete(ctx context.Context, ref *provider.Reference) error { bodyStr, err := json.Marshal(ref) diff --git a/pkg/storage/fs/owncloud/owncloud.go b/pkg/storage/fs/owncloud/owncloud.go index 51a1299ec3..51a5ef8b05 100644 --- a/pkg/storage/fs/owncloud/owncloud.go +++ b/pkg/storage/fs/owncloud/owncloud.go @@ -1186,6 +1186,11 @@ func (fs *ocfs) CreateDir(ctx context.Context, ref *provider.Reference) (err err return fs.propagate(ctx, ip) } +// TouchFile as defined in the storage.FS interface +func (fs *ocfs) TouchFile(ctx context.Context, ref *provider.Reference) error { + return fmt.Errorf("unimplemented: TouchFile") +} + func (fs *ocfs) isShareFolderChild(sp string) bool { return strings.HasPrefix(sp, fs.c.ShareFolder) } diff --git a/pkg/storage/fs/owncloudsql/owncloudsql.go b/pkg/storage/fs/owncloudsql/owncloudsql.go index 43db8f3b89..05b929330f 100644 --- a/pkg/storage/fs/owncloudsql/owncloudsql.go +++ b/pkg/storage/fs/owncloudsql/owncloudsql.go @@ -766,6 +766,11 @@ func (fs *owncloudsqlfs) CreateDir(ctx context.Context, ref *provider.Reference) return fs.propagate(ctx, filepath.Dir(ip)) } +// TouchFile as defined in the storage.FS interface +func (fs *owncloudsqlfs) TouchFile(ctx context.Context, ref *provider.Reference) error { + return fmt.Errorf("unimplemented: TouchFile") +} + func (fs *owncloudsqlfs) CreateReference(ctx context.Context, sp string, targetURI *url.URL) error { return errtypes.NotSupported("owncloudsql: operation not supported") } diff --git a/pkg/storage/fs/s3/s3.go b/pkg/storage/fs/s3/s3.go index 39d9fb7d6a..330706aae0 100644 --- a/pkg/storage/fs/s3/s3.go +++ b/pkg/storage/fs/s3/s3.go @@ -326,6 +326,11 @@ func (fs *s3FS) CreateDir(ctx context.Context, ref *provider.Reference) error { return nil } +// TouchFile as defined in the storage.FS interface +func (fs *s3FS) TouchFile(ctx context.Context, ref *provider.Reference) error { + return fmt.Errorf("unimplemented: TouchFile") +} + func (fs *s3FS) Delete(ctx context.Context, ref *provider.Reference) error { log := appctx.GetLogger(ctx) diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index f43a938152..8b61b6f616 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -32,6 +32,7 @@ type FS interface { GetHome(ctx context.Context) (string, error) CreateHome(ctx context.Context) error CreateDir(ctx context.Context, ref *provider.Reference) error + TouchFile(ctx context.Context, ref *provider.Reference) error Delete(ctx context.Context, ref *provider.Reference) error Move(ctx context.Context, oldRef, newRef *provider.Reference) error GetMD(ctx context.Context, ref *provider.Reference, mdKeys []string) (*provider.ResourceInfo, error) diff --git a/pkg/storage/utils/decomposedfs/decomposedfs.go b/pkg/storage/utils/decomposedfs/decomposedfs.go index 4ebb5c3cb2..6b968febf2 100644 --- a/pkg/storage/utils/decomposedfs/decomposedfs.go +++ b/pkg/storage/utils/decomposedfs/decomposedfs.go @@ -23,6 +23,7 @@ package decomposedfs import ( "context" + "fmt" "io" "net/url" "os" @@ -320,6 +321,11 @@ func (fs *Decomposedfs) CreateDir(ctx context.Context, ref *provider.Reference) return } +// TouchFile as defined in the storage.FS interface +func (fs *Decomposedfs) TouchFile(ctx context.Context, ref *provider.Reference) error { + return fmt.Errorf("unimplemented: TouchFile") +} + // CreateReference creates a reference as a node folder with the target stored in extended attributes // There is no difference between the /Shares folder and normal nodes because the storage is not supposed to be accessible without the storage provider. // In effect everything is a shadow namespace. diff --git a/pkg/storage/utils/eosfs/eosfs.go b/pkg/storage/utils/eosfs/eosfs.go index 3d045dd553..ce62dde0c5 100644 --- a/pkg/storage/utils/eosfs/eosfs.go +++ b/pkg/storage/utils/eosfs/eosfs.go @@ -1214,6 +1214,29 @@ func (fs *eosfs) CreateDir(ctx context.Context, ref *provider.Reference) error { return fs.c.CreateDir(ctx, auth, fn) } +// TouchFile as defined in the storage.FS interface +func (fs *eosfs) TouchFile(ctx context.Context, ref *provider.Reference) error { + log := appctx.GetLogger(ctx) + u, err := getUser(ctx) + if err != nil { + return errors.Wrap(err, "eosfs: no user in ctx") + } + p, err := fs.resolve(ctx, ref) + if err != nil { + return nil + } + + auth, err := fs.getUserAuth(ctx, u, p) + if err != nil { + return err + } + + log.Info().Msgf("eosfs: touch file: path=%s", p) + + fn := fs.wrap(ctx, p) + return fs.c.Touch(ctx, auth, fn) +} + func (fs *eosfs) CreateReference(ctx context.Context, p string, targetURI *url.URL) error { // TODO(labkode): for the time being we only allow creating references // in the virtual share folder to not pollute the nominal user tree. diff --git a/pkg/storage/utils/localfs/localfs.go b/pkg/storage/utils/localfs/localfs.go index 9f807d9ad2..4dc7c37e78 100644 --- a/pkg/storage/utils/localfs/localfs.go +++ b/pkg/storage/utils/localfs/localfs.go @@ -782,6 +782,11 @@ func (fs *localfs) CreateDir(ctx context.Context, ref *provider.Reference) error return fs.propagate(ctx, path.Dir(fn)) } +// TouchFile as defined in the storage.FS interface +func (fs *localfs) TouchFile(ctx context.Context, ref *provider.Reference) error { + return fmt.Errorf("unimplemented: TouchFile") +} + func (fs *localfs) Delete(ctx context.Context, ref *provider.Reference) error { fn, err := fs.resolve(ctx, ref) if err != nil { From 71962293fceb7631cf41a22420556b3c522929e2 Mon Sep 17 00:00:00 2001 From: Willy Kloucek <34452982+wkloucek@users.noreply.github.com> Date: Fri, 17 Dec 2021 12:48:00 +0100 Subject: [PATCH 24/58] Dummy implementation of the Lock CS3APIs (#2350) --- ...ement-add-locking-methods-to-interfaces.md | 8 ++ go.mod | 2 +- go.sum | 8 +- .../grpc/services/gateway/storageprovider.go | 84 +++++++++++- .../publicstorageprovider.go | 20 +++ .../storageprovider/storageprovider.go | 126 ++++++++++++++++++ pkg/storage/fs/nextcloud/nextcloud.go | 20 +++ pkg/storage/fs/owncloud/owncloud.go | 20 +++ pkg/storage/fs/owncloudsql/owncloudsql.go | 20 +++ pkg/storage/fs/s3/s3.go | 20 +++ pkg/storage/storage.go | 4 + .../utils/decomposedfs/decomposedfs.go | 20 +++ pkg/storage/utils/eosfs/eosfs.go | 20 +++ pkg/storage/utils/localfs/localfs.go | 20 +++ 14 files changed, 383 insertions(+), 9 deletions(-) create mode 100644 changelog/unreleased/enhancement-add-locking-methods-to-interfaces.md diff --git a/changelog/unreleased/enhancement-add-locking-methods-to-interfaces.md b/changelog/unreleased/enhancement-add-locking-methods-to-interfaces.md new file mode 100644 index 0000000000..edf8e4b902 --- /dev/null +++ b/changelog/unreleased/enhancement-add-locking-methods-to-interfaces.md @@ -0,0 +1,8 @@ +Enhancement: add file locking methods to the storage and filesystem interfaces + +We've added the file locking methods from the CS3apis to the storage and filesystem +interfaces. As of now they are dummy implementations and will only return "unimplemented" +errors. + +https://github.com/cs3org/reva/pull/2350 +https://github.com/cs3org/cs3apis/pull/160 diff --git a/go.mod b/go.mod index b5b9d08395..a0c8c8cd0f 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/cheggaaa/pb v1.0.29 github.com/coreos/go-oidc v2.2.1+incompatible github.com/cs3org/cato v0.0.0-20200828125504-e418fc54dd5e - github.com/cs3org/go-cs3apis v0.0.0-20211213090556-12c0d565f51d + github.com/cs3org/go-cs3apis v0.0.0-20211214102047-7ce3134d7bf8 github.com/cubewise-code/go-mime v0.0.0-20200519001935-8c5762b177d8 github.com/eventials/go-tus v0.0.0-20200718001131-45c7ec8f5d59 github.com/gdexlab/go-render v1.0.1 diff --git a/go.sum b/go.sum index c937822ad9..73d811a43a 100644 --- a/go.sum +++ b/go.sum @@ -111,12 +111,8 @@ github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSV github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cs3org/cato v0.0.0-20200828125504-e418fc54dd5e h1:tqSPWQeueWTKnJVMJffz4pz0o1WuQxJ28+5x5JgaHD8= github.com/cs3org/cato v0.0.0-20200828125504-e418fc54dd5e/go.mod h1:XJEZ3/EQuI3BXTp/6DUzFr850vlxq11I6satRtz0YQ4= -github.com/cs3org/go-cs3apis v0.0.0-20210325133324-32b03d75a535 h1:555D8A3ddKqb4OyK9v5mdphw2zDLWKGXOkcnf1RQwTA= -github.com/cs3org/go-cs3apis v0.0.0-20210325133324-32b03d75a535/go.mod h1:UXha4TguuB52H14EMoSsCqDj7k8a/t7g4gVP+bgY5LY= -github.com/cs3org/go-cs3apis v0.0.0-20211104090126-8e972dca8304 h1:e/nIPR518vyvrulo9goAZTtYD6gFfu/2/9MDe6mTGcw= -github.com/cs3org/go-cs3apis v0.0.0-20211104090126-8e972dca8304/go.mod h1:UXha4TguuB52H14EMoSsCqDj7k8a/t7g4gVP+bgY5LY= -github.com/cs3org/go-cs3apis v0.0.0-20211213090556-12c0d565f51d h1:gnb2ciU4N+RwUug/nwe54wenWi7vSp5bAAjXINlgHZ8= -github.com/cs3org/go-cs3apis v0.0.0-20211213090556-12c0d565f51d/go.mod h1:UXha4TguuB52H14EMoSsCqDj7k8a/t7g4gVP+bgY5LY= +github.com/cs3org/go-cs3apis v0.0.0-20211214102047-7ce3134d7bf8 h1:PqOprF37OvwCbAN5W23znknGk6N/LMayqLAeP904FHE= +github.com/cs3org/go-cs3apis v0.0.0-20211214102047-7ce3134d7bf8/go.mod h1:UXha4TguuB52H14EMoSsCqDj7k8a/t7g4gVP+bgY5LY= github.com/cubewise-code/go-mime v0.0.0-20200519001935-8c5762b177d8 h1:Z9lwXumT5ACSmJ7WGnFl+OMLLjpz5uR2fyz7dC255FI= github.com/cubewise-code/go-mime v0.0.0-20200519001935-8c5762b177d8/go.mod h1:4abs/jPXcmJzYoYGF91JF9Uq9s/KL5n1jvFDix8KcqY= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= diff --git a/internal/grpc/services/gateway/storageprovider.go b/internal/grpc/services/gateway/storageprovider.go index cdd622e9c7..54c0230f9b 100644 --- a/internal/grpc/services/gateway/storageprovider.go +++ b/internal/grpc/services/gateway/storageprovider.go @@ -1201,7 +1201,7 @@ func (s *svc) SetArbitraryMetadata(ctx context.Context, req *provider.SetArbitra if gstatus.Code(err) == codes.PermissionDenied { return &provider.SetArbitraryMetadataResponse{Status: &rpc.Status{Code: rpc.Code_CODE_PERMISSION_DENIED}}, nil } - return nil, errors.Wrap(err, "gateway: error calling Stat") + return nil, errors.Wrap(err, "gateway: error calling SetArbitraryMetadata") } return res, nil @@ -1221,7 +1221,87 @@ func (s *svc) UnsetArbitraryMetadata(ctx context.Context, req *provider.UnsetArb if gstatus.Code(err) == codes.PermissionDenied { return &provider.UnsetArbitraryMetadataResponse{Status: &rpc.Status{Code: rpc.Code_CODE_PERMISSION_DENIED}}, nil } - return nil, errors.Wrap(err, "gateway: error calling Stat") + return nil, errors.Wrap(err, "gateway: error calling UnsetArbitraryMetadata") + } + + return res, nil +} + +// SetLock puts a lock on the given reference +func (s *svc) SetLock(ctx context.Context, req *provider.SetLockRequest) (*provider.SetLockResponse, error) { + c, err := s.find(ctx, req.Ref) + if err != nil { + return &provider.SetLockResponse{ + Status: status.NewStatusFromErrType(ctx, "SetLock ref="+req.Ref.String(), err), + }, nil + } + + res, err := c.SetLock(ctx, req) + if err != nil { + if gstatus.Code(err) == codes.PermissionDenied { + return &provider.SetLockResponse{Status: &rpc.Status{Code: rpc.Code_CODE_PERMISSION_DENIED}}, nil + } + return nil, errors.Wrap(err, "gateway: error calling SetLock") + } + + return res, nil +} + +// GetLock returns an existing lock on the given reference +func (s *svc) GetLock(ctx context.Context, req *provider.GetLockRequest) (*provider.GetLockResponse, error) { + c, err := s.find(ctx, req.Ref) + if err != nil { + return &provider.GetLockResponse{ + Status: status.NewStatusFromErrType(ctx, "GetLock ref="+req.Ref.String(), err), + }, nil + } + + res, err := c.GetLock(ctx, req) + if err != nil { + if gstatus.Code(err) == codes.PermissionDenied { + return &provider.GetLockResponse{Status: &rpc.Status{Code: rpc.Code_CODE_PERMISSION_DENIED}}, nil + } + return nil, errors.Wrap(err, "gateway: error calling GetLock") + } + + return res, nil +} + +// RefreshLock refreshes an existing lock on the given reference +func (s *svc) RefreshLock(ctx context.Context, req *provider.RefreshLockRequest) (*provider.RefreshLockResponse, error) { + c, err := s.find(ctx, req.Ref) + if err != nil { + return &provider.RefreshLockResponse{ + Status: status.NewStatusFromErrType(ctx, "RefreshLock ref="+req.Ref.String(), err), + }, nil + } + + res, err := c.RefreshLock(ctx, req) + if err != nil { + if gstatus.Code(err) == codes.PermissionDenied { + return &provider.RefreshLockResponse{Status: &rpc.Status{Code: rpc.Code_CODE_PERMISSION_DENIED}}, nil + } + return nil, errors.Wrap(err, "gateway: error calling RefreshLock") + } + + return res, nil +} + +// Unlock removes an existing lock from the given reference +func (s *svc) Unlock(ctx context.Context, req *provider.UnlockRequest) (*provider.UnlockResponse, error) { + c, err := s.find(ctx, req.Ref) + if err != nil { + return &provider.UnlockResponse{ + Status: status.NewStatusFromErrType(ctx, "Unlock ref="+req.Ref.String(), err), + }, nil + } + + res, err := c.Unlock(ctx, req) + if err != nil { + if gstatus.Code(err) == codes.PermissionDenied { + return &provider.UnlockResponse{Status: &rpc.Status{Code: rpc.Code_CODE_PERMISSION_DENIED}}, nil + } + return nil, errors.Wrap(err, "gateway: error calling Unlock") } return res, nil diff --git a/internal/grpc/services/publicstorageprovider/publicstorageprovider.go b/internal/grpc/services/publicstorageprovider/publicstorageprovider.go index 80b64f4e99..9aa3aed324 100644 --- a/internal/grpc/services/publicstorageprovider/publicstorageprovider.go +++ b/internal/grpc/services/publicstorageprovider/publicstorageprovider.go @@ -124,6 +124,26 @@ func (s *service) UnsetArbitraryMetadata(ctx context.Context, req *provider.Unse return nil, gstatus.Errorf(codes.Unimplemented, "method not implemented") } +// SetLock puts a lock on the given reference +func (s *service) SetLock(ctx context.Context, req *provider.SetLockRequest) (*provider.SetLockResponse, error) { + return nil, gstatus.Errorf(codes.Unimplemented, "method not implemented") +} + +// GetLock returns an existing lock on the given reference +func (s *service) GetLock(ctx context.Context, req *provider.GetLockRequest) (*provider.GetLockResponse, error) { + return nil, gstatus.Errorf(codes.Unimplemented, "method not implemented") +} + +// RefreshLock refreshes an existing lock on the given reference +func (s *service) RefreshLock(ctx context.Context, req *provider.RefreshLockRequest) (*provider.RefreshLockResponse, error) { + return nil, gstatus.Errorf(codes.Unimplemented, "method not implemented") +} + +// Unlock removes an existing lock from the given reference +func (s *service) Unlock(ctx context.Context, req *provider.UnlockRequest) (*provider.UnlockResponse, error) { + return nil, gstatus.Errorf(codes.Unimplemented, "method not implemented") +} + func (s *service) InitiateFileDownload(ctx context.Context, req *provider.InitiateFileDownloadRequest) (*provider.InitiateFileDownloadResponse, error) { statReq := &provider.StatRequest{Ref: req.Ref} statRes, err := s.Stat(ctx, statReq) diff --git a/internal/grpc/services/storageprovider/storageprovider.go b/internal/grpc/services/storageprovider/storageprovider.go index a0d659005a..13dc51ed15 100644 --- a/internal/grpc/services/storageprovider/storageprovider.go +++ b/internal/grpc/services/storageprovider/storageprovider.go @@ -264,6 +264,132 @@ func (s *service) UnsetArbitraryMetadata(ctx context.Context, req *provider.Unse return res, nil } +// SetLock puts a lock on the given reference +func (s *service) SetLock(ctx context.Context, req *provider.SetLockRequest) (*provider.SetLockResponse, error) { + newRef, err := s.unwrap(ctx, req.Ref) + if err != nil { + err := errors.Wrap(err, "storageprovidersvc: error unwrapping path") + return &provider.SetLockResponse{ + Status: status.NewInternal(ctx, err, "error setting lock"), + }, nil + } + + if err := s.storage.SetLock(ctx, newRef, req.Lock); err != nil { + var st *rpc.Status + switch err.(type) { + case errtypes.IsNotFound: + st = status.NewNotFound(ctx, "path not found when setting lock") + case errtypes.PermissionDenied: + st = status.NewPermissionDenied(ctx, err, "permission denied") + default: + st = status.NewInternal(ctx, err, "error setting lock: "+req.Ref.String()) + } + return &provider.SetLockResponse{ + Status: st, + }, nil + } + + res := &provider.SetLockResponse{ + Status: status.NewOK(ctx), + } + return res, nil +} + +// GetLock returns an existing lock on the given reference +func (s *service) GetLock(ctx context.Context, req *provider.GetLockRequest) (*provider.GetLockResponse, error) { + newRef, err := s.unwrap(ctx, req.Ref) + if err != nil { + err := errors.Wrap(err, "storageprovidersvc: error unwrapping path") + return &provider.GetLockResponse{ + Status: status.NewInternal(ctx, err, "error getting lock"), + }, nil + } + + var lock *provider.Lock + if lock, err = s.storage.GetLock(ctx, newRef); err != nil { + var st *rpc.Status + switch err.(type) { + case errtypes.IsNotFound: + st = status.NewNotFound(ctx, "path not found when getting lock") + case errtypes.PermissionDenied: + st = status.NewPermissionDenied(ctx, err, "permission denied") + default: + st = status.NewInternal(ctx, err, "error getting lock: "+req.Ref.String()) + } + return &provider.GetLockResponse{ + Status: st, + }, nil + } + + res := &provider.GetLockResponse{ + Status: status.NewOK(ctx), + Lock: lock, + } + return res, nil +} + +// RefreshLock refreshes an existing lock on the given reference +func (s *service) RefreshLock(ctx context.Context, req *provider.RefreshLockRequest) (*provider.RefreshLockResponse, error) { + newRef, err := s.unwrap(ctx, req.Ref) + if err != nil { + err := errors.Wrap(err, "storageprovidersvc: error unwrapping path") + return &provider.RefreshLockResponse{ + Status: status.NewInternal(ctx, err, "error refreshing lock"), + }, nil + } + + if err = s.storage.RefreshLock(ctx, newRef, req.Lock); err != nil { + var st *rpc.Status + switch err.(type) { + case errtypes.IsNotFound: + st = status.NewNotFound(ctx, "path not found when refreshing lock") + case errtypes.PermissionDenied: + st = status.NewPermissionDenied(ctx, err, "permission denied") + default: + st = status.NewInternal(ctx, err, "error refreshing lock: "+req.Ref.String()) + } + return &provider.RefreshLockResponse{ + Status: st, + }, nil + } + + res := &provider.RefreshLockResponse{ + Status: status.NewOK(ctx), + } + return res, nil +} + +// Unlock removes an existing lock from the given reference +func (s *service) Unlock(ctx context.Context, req *provider.UnlockRequest) (*provider.UnlockResponse, error) { + newRef, err := s.unwrap(ctx, req.Ref) + if err != nil { + err := errors.Wrap(err, "storageprovidersvc: error unwrapping path") + return &provider.UnlockResponse{ + Status: status.NewInternal(ctx, err, "error on unlocking"), + }, nil + } + + if err = s.storage.Unlock(ctx, newRef); err != nil { + var st *rpc.Status + switch err.(type) { + case errtypes.IsNotFound: + st = status.NewNotFound(ctx, "path not found when unlocking") + case errtypes.PermissionDenied: + st = status.NewPermissionDenied(ctx, err, "permission denied") + default: + st = status.NewInternal(ctx, err, "error unlocking: "+req.Ref.String()) + } + return &provider.UnlockResponse{ + Status: st, + }, nil + } + + res := &provider.UnlockResponse{ + Status: status.NewOK(ctx), + } + return res, nil +} + func (s *service) InitiateFileDownload(ctx context.Context, req *provider.InitiateFileDownloadRequest) (*provider.InitiateFileDownloadResponse, error) { // TODO(labkode): maybe add some checks before download starts? eg. check permissions? // TODO(labkode): maybe add short-lived token? diff --git a/pkg/storage/fs/nextcloud/nextcloud.go b/pkg/storage/fs/nextcloud/nextcloud.go index 567a880863..04e5b54912 100644 --- a/pkg/storage/fs/nextcloud/nextcloud.go +++ b/pkg/storage/fs/nextcloud/nextcloud.go @@ -762,6 +762,26 @@ func (nc *StorageDriver) UnsetArbitraryMetadata(ctx context.Context, ref *provid return err } +// GetLock returns an existing lock on the given reference +func (nc *StorageDriver) GetLock(ctx context.Context, ref *provider.Reference) (*provider.Lock, error) { + return nil, errtypes.NotSupported("unimplemented") +} + +// SetLock puts a lock on the given reference +func (nc *StorageDriver) SetLock(ctx context.Context, ref *provider.Reference, lock *provider.Lock) error { + return errtypes.NotSupported("unimplemented") +} + +// RefreshLock refreshes an existing lock on the given reference +func (nc *StorageDriver) RefreshLock(ctx context.Context, ref *provider.Reference, lock *provider.Lock) error { + return errtypes.NotSupported("unimplemented") +} + +// Unlock removes an existing lock from the given reference +func (nc *StorageDriver) Unlock(ctx context.Context, ref *provider.Reference) error { + return errtypes.NotSupported("unimplemented") +} + // ListStorageSpaces as defined in the storage.FS interface func (nc *StorageDriver) ListStorageSpaces(ctx context.Context, f []*provider.ListStorageSpacesRequest_Filter, _ map[string]struct{}) ([]*provider.StorageSpace, error) { bodyStr, _ := json.Marshal(f) diff --git a/pkg/storage/fs/owncloud/owncloud.go b/pkg/storage/fs/owncloud/owncloud.go index 51a5ef8b05..5e0a65bce8 100644 --- a/pkg/storage/fs/owncloud/owncloud.go +++ b/pkg/storage/fs/owncloud/owncloud.go @@ -1473,6 +1473,26 @@ func (fs *ocfs) UnsetArbitraryMetadata(ctx context.Context, ref *provider.Refere } } +// GetLock returns an existing lock on the given reference +func (fs *ocfs) GetLock(ctx context.Context, ref *provider.Reference) (*provider.Lock, error) { + return nil, errtypes.NotSupported("unimplemented") +} + +// SetLock puts a lock on the given reference +func (fs *ocfs) SetLock(ctx context.Context, ref *provider.Reference, lock *provider.Lock) error { + return errtypes.NotSupported("unimplemented") +} + +// RefreshLock refreshes an existing lock on the given reference +func (fs *ocfs) RefreshLock(ctx context.Context, ref *provider.Reference, lock *provider.Lock) error { + return errtypes.NotSupported("unimplemented") +} + +// Unlock removes an existing lock from the given reference +func (fs *ocfs) Unlock(ctx context.Context, ref *provider.Reference) error { + return errtypes.NotSupported("unimplemented") +} + // Delete is actually only a move to trash // // This is a first optimistic approach. diff --git a/pkg/storage/fs/owncloudsql/owncloudsql.go b/pkg/storage/fs/owncloudsql/owncloudsql.go index 05b929330f..64803fc945 100644 --- a/pkg/storage/fs/owncloudsql/owncloudsql.go +++ b/pkg/storage/fs/owncloudsql/owncloudsql.go @@ -1024,6 +1024,26 @@ func (fs *owncloudsqlfs) UnsetArbitraryMetadata(ctx context.Context, ref *provid } } +// GetLock returns an existing lock on the given reference +func (fs *owncloudsqlfs) GetLock(ctx context.Context, ref *provider.Reference) (*provider.Lock, error) { + return nil, errtypes.NotSupported("unimplemented") +} + +// SetLock puts a lock on the given reference +func (fs *owncloudsqlfs) SetLock(ctx context.Context, ref *provider.Reference, lock *provider.Lock) error { + return errtypes.NotSupported("unimplemented") +} + +// RefreshLock refreshes an existing lock on the given reference +func (fs *owncloudsqlfs) RefreshLock(ctx context.Context, ref *provider.Reference, lock *provider.Lock) error { + return errtypes.NotSupported("unimplemented") +} + +// Unlock removes an existing lock from the given reference +func (fs *owncloudsqlfs) Unlock(ctx context.Context, ref *provider.Reference) error { + return errtypes.NotSupported("unimplemented") +} + // Delete is actually only a move to trash // // This is a first optimistic approach. diff --git a/pkg/storage/fs/s3/s3.go b/pkg/storage/fs/s3/s3.go index 330706aae0..063800326e 100644 --- a/pkg/storage/fs/s3/s3.go +++ b/pkg/storage/fs/s3/s3.go @@ -280,6 +280,26 @@ func (fs *s3FS) UnsetArbitraryMetadata(ctx context.Context, ref *provider.Refere return errtypes.NotSupported("s3: operation not supported") } +// GetLock returns an existing lock on the given reference +func (fs *s3FS) GetLock(ctx context.Context, ref *provider.Reference) (*provider.Lock, error) { + return nil, errtypes.NotSupported("unimplemented") +} + +// SetLock puts a lock on the given reference +func (fs *s3FS) SetLock(ctx context.Context, ref *provider.Reference, lock *provider.Lock) error { + return errtypes.NotSupported("unimplemented") +} + +// RefreshLock refreshes an existing lock on the given reference +func (fs *s3FS) RefreshLock(ctx context.Context, ref *provider.Reference, lock *provider.Lock) error { + return errtypes.NotSupported("unimplemented") +} + +// Unlock removes an existing lock from the given reference +func (fs *s3FS) Unlock(ctx context.Context, ref *provider.Reference) error { + return errtypes.NotSupported("unimplemented") +} + func (fs *s3FS) CreateReference(ctx context.Context, path string, targetURI *url.URL) error { // TODO(jfd):implement return errtypes.NotSupported("s3: operation not supported") diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index 8b61b6f616..4256d10cdf 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -58,6 +58,10 @@ type FS interface { Shutdown(ctx context.Context) error SetArbitraryMetadata(ctx context.Context, ref *provider.Reference, md *provider.ArbitraryMetadata) error UnsetArbitraryMetadata(ctx context.Context, ref *provider.Reference, keys []string) error + SetLock(ctx context.Context, ref *provider.Reference, lock *provider.Lock) error + GetLock(ctx context.Context, ref *provider.Reference) (*provider.Lock, error) + RefreshLock(ctx context.Context, ref *provider.Reference, lock *provider.Lock) error + Unlock(ctx context.Context, ref *provider.Reference) error ListStorageSpaces(ctx context.Context, filter []*provider.ListStorageSpacesRequest_Filter, permissions map[string]struct{}) ([]*provider.StorageSpace, error) CreateStorageSpace(ctx context.Context, req *provider.CreateStorageSpaceRequest) (*provider.CreateStorageSpaceResponse, error) UpdateStorageSpace(ctx context.Context, req *provider.UpdateStorageSpaceRequest) (*provider.UpdateStorageSpaceResponse, error) diff --git a/pkg/storage/utils/decomposedfs/decomposedfs.go b/pkg/storage/utils/decomposedfs/decomposedfs.go index 6b968febf2..5e40163e6e 100644 --- a/pkg/storage/utils/decomposedfs/decomposedfs.go +++ b/pkg/storage/utils/decomposedfs/decomposedfs.go @@ -523,3 +523,23 @@ func (fs *Decomposedfs) Download(ctx context.Context, ref *provider.Reference) ( } return reader, nil } + +// GetLock returns an existing lock on the given reference +func (fs *Decomposedfs) GetLock(ctx context.Context, ref *provider.Reference) (*provider.Lock, error) { + return nil, errtypes.NotSupported("unimplemented") +} + +// SetLock puts a lock on the given reference +func (fs *Decomposedfs) SetLock(ctx context.Context, ref *provider.Reference, lock *provider.Lock) error { + return errtypes.NotSupported("unimplemented") +} + +// RefreshLock refreshes an existing lock on the given reference +func (fs *Decomposedfs) RefreshLock(ctx context.Context, ref *provider.Reference, lock *provider.Lock) error { + return errtypes.NotSupported("unimplemented") +} + +// Unlock removes an existing lock from the given reference +func (fs *Decomposedfs) Unlock(ctx context.Context, ref *provider.Reference) error { + return errtypes.NotSupported("unimplemented") +} diff --git a/pkg/storage/utils/eosfs/eosfs.go b/pkg/storage/utils/eosfs/eosfs.go index ce62dde0c5..89cd109cdc 100644 --- a/pkg/storage/utils/eosfs/eosfs.go +++ b/pkg/storage/utils/eosfs/eosfs.go @@ -535,6 +535,26 @@ func (fs *eosfs) UnsetArbitraryMetadata(ctx context.Context, ref *provider.Refer return nil } +// GetLock returns an existing lock on the given reference +func (fs *eosfs) GetLock(ctx context.Context, ref *provider.Reference) (*provider.Lock, error) { + return nil, errtypes.NotSupported("unimplemented") +} + +// SetLock puts a lock on the given reference +func (fs *eosfs) SetLock(ctx context.Context, ref *provider.Reference, lock *provider.Lock) error { + return errtypes.NotSupported("unimplemented") +} + +// RefreshLock refreshes an existing lock on the given reference +func (fs *eosfs) RefreshLock(ctx context.Context, ref *provider.Reference, lock *provider.Lock) error { + return errtypes.NotSupported("unimplemented") +} + +// Unlock removes an existing lock from the given reference +func (fs *eosfs) Unlock(ctx context.Context, ref *provider.Reference) error { + return errtypes.NotSupported("unimplemented") +} + func (fs *eosfs) AddGrant(ctx context.Context, ref *provider.Reference, g *provider.Grant) error { u, err := getUser(ctx) if err != nil { diff --git a/pkg/storage/utils/localfs/localfs.go b/pkg/storage/utils/localfs/localfs.go index 4dc7c37e78..af70f404e3 100644 --- a/pkg/storage/utils/localfs/localfs.go +++ b/pkg/storage/utils/localfs/localfs.go @@ -711,6 +711,26 @@ func (fs *localfs) UnsetArbitraryMetadata(ctx context.Context, ref *provider.Ref return fs.propagate(ctx, np) } +// GetLock returns an existing lock on the given reference +func (fs *localfs) GetLock(ctx context.Context, ref *provider.Reference) (*provider.Lock, error) { + return nil, errtypes.NotSupported("unimplemented") +} + +// SetLock puts a lock on the given reference +func (fs *localfs) SetLock(ctx context.Context, ref *provider.Reference, lock *provider.Lock) error { + return errtypes.NotSupported("unimplemented") +} + +// RefreshLock refreshes an existing lock on the given reference +func (fs *localfs) RefreshLock(ctx context.Context, ref *provider.Reference, lock *provider.Lock) error { + return errtypes.NotSupported("unimplemented") +} + +// Unlock removes an existing lock from the given reference +func (fs *localfs) Unlock(ctx context.Context, ref *provider.Reference) error { + return errtypes.NotSupported("unimplemented") +} + func (fs *localfs) GetHome(ctx context.Context) (string, error) { if fs.conf.DisableHome { return "", errtypes.NotSupported("local: get home not supported") From 9c822d0416360a3fc79d1b6c084d6cbd7d55a702 Mon Sep 17 00:00:00 2001 From: Willy Kloucek <34452982+wkloucek@users.noreply.github.com> Date: Mon, 20 Dec 2021 10:07:30 +0100 Subject: [PATCH 25/58] allow new file create with app provider on public links (#2385) --- changelog/unreleased/enhancement-app-new-public-link.md | 5 +++++ pkg/auth/scope/resourceinfo.go | 1 + 2 files changed, 6 insertions(+) create mode 100644 changelog/unreleased/enhancement-app-new-public-link.md diff --git a/changelog/unreleased/enhancement-app-new-public-link.md b/changelog/unreleased/enhancement-app-new-public-link.md new file mode 100644 index 0000000000..01e315441b --- /dev/null +++ b/changelog/unreleased/enhancement-app-new-public-link.md @@ -0,0 +1,5 @@ +Enhancement: Allow to create new files with the app provider on public links + +We've added the option to create files with the app provider on public links. + +https://github.com/cs3org/reva/pull/2385 diff --git a/pkg/auth/scope/resourceinfo.go b/pkg/auth/scope/resourceinfo.go index 1a6c41a40f..d2ad4240fc 100644 --- a/pkg/auth/scope/resourceinfo.go +++ b/pkg/auth/scope/resourceinfo.go @@ -102,6 +102,7 @@ func checkResourcePath(path string) bool { "/dataprovider", "/data", "/app/open", + "/app/new", "/archiver", "/ocs/v2.php/cloud/capabilities", "/ocs/v1.php/cloud/capabilities", From cd9a6fb1971afd3d9b436d4f9cbfcd35da03b2b7 Mon Sep 17 00:00:00 2001 From: Phil Davis Date: Wed, 22 Dec 2021 19:45:38 +0545 Subject: [PATCH 26/58] Bump core commit id and use core master for tests (#2391) --- .drone.env | 4 ++-- tests/acceptance/expected-failures-on-OCIS-storage.md | 10 ++++++++-- tests/acceptance/expected-failures-on-S3NG-storage.md | 10 ++++++++-- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/.drone.env b/.drone.env index b07b2f80cc..1c52a407c5 100644 --- a/.drone.env +++ b/.drone.env @@ -1,3 +1,3 @@ # The test runner source for API tests -CORE_COMMITID=38ee68e6358443e980ba5e7036cf1fb554443814 -CORE_BRANCH=acceptance-test-changes-waiting-2021-11 +CORE_COMMITID=fe2504387a9e914e03d6af55c54e34a617e55cf0 +CORE_BRANCH=master diff --git a/tests/acceptance/expected-failures-on-OCIS-storage.md b/tests/acceptance/expected-failures-on-OCIS-storage.md index a509283234..ab20e85f30 100644 --- a/tests/acceptance/expected-failures-on-OCIS-storage.md +++ b/tests/acceptance/expected-failures-on-OCIS-storage.md @@ -1185,8 +1185,8 @@ Not everything needs to be implemented for ocis. While the oc10 testsuite covers - [apiWebdavProperties1/copyFile.feature:108](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavProperties1/copyFile.feature#L108) - [apiWebdavProperties1/createFolder.feature:71](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavProperties1/createFolder.feature#L71) - [apiWebdavProperties1/createFolder.feature:72](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavProperties1/createFolder.feature#L72) -- [apiWebdavUpload1/uploadFile.feature:127](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavUpload1/uploadFile.feature#L127) -- [apiWebdavUpload1/uploadFile.feature:128](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavUpload1/uploadFile.feature#L128) +- [apiWebdavUpload1/uploadFile.feature:132](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavUpload1/uploadFile.feature#L132) +- [apiWebdavUpload1/uploadFile.feature:133](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavUpload1/uploadFile.feature#L133) - [apiWebdavUpload2/uploadFileToBlacklistedNameUsingOldChunking.feature:19](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavUpload2/uploadFileToBlacklistedNameUsingOldChunking.feature#L19) - [apiWebdavUpload2/uploadFileToBlacklistedNameUsingOldChunking.feature:35](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavUpload2/uploadFileToBlacklistedNameUsingOldChunking.feature#L35) - [apiWebdavUpload2/uploadFileToBlacklistedNameUsingOldChunking.feature:36](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavUpload2/uploadFileToBlacklistedNameUsingOldChunking.feature#L36) @@ -1342,6 +1342,9 @@ _ocs: api compatibility, return correct status code_ - [apiVersions/fileVersions.feature:455](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiVersions/fileVersions.feature#L455) - [apiVersions/fileVersionsSharingToShares.feature:305](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiVersions/fileVersionsSharingToShares.feature#L305) +#### [file versions do not report the version author](https://github.com/owncloud/ocis/issues/2914) +- [apiVersions/fileVersions.feature:474](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiVersions/fileVersions.feature#L474) + ### [Allow public link sharing only for certain groups feature not implemented] - [apiSharePublicLink2/allowGroupToCreatePublicLinks.feature:35](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiSharePublicLink2/allowGroupToCreatePublicLinks.feature#L35) - [apiSharePublicLink2/allowGroupToCreatePublicLinks.feature:91](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiSharePublicLink2/allowGroupToCreatePublicLinks.feature#L91) @@ -1365,3 +1368,6 @@ _ocs: api compatibility, return correct status code_ ### [Content-type is not multipart/byteranges when downloading file with Range Header](https://github.com/owncloud/ocis/issues/2677) - [apiWebdavOperations/downloadFile.feature:169](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavOperations/downloadFile.feature#L169) - [apiWebdavOperations/downloadFile.feature:170](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavOperations/downloadFile.feature#L170) + +### [Calculate personal space id in tests when running on reva](https://github.com/owncloud/core/issues/39617) +- [apiWebdavUpload1/uploadFile.feature:32](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavUpload1/uploadFile.feature#L32) diff --git a/tests/acceptance/expected-failures-on-S3NG-storage.md b/tests/acceptance/expected-failures-on-S3NG-storage.md index 9c7bfff96b..684c92e08d 100644 --- a/tests/acceptance/expected-failures-on-S3NG-storage.md +++ b/tests/acceptance/expected-failures-on-S3NG-storage.md @@ -16,6 +16,9 @@ Basic file management like up and download, move, copy, properties, quota, trash - [apiVersions/fileVersions.feature:455](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiVersions/fileVersions.feature#L455) - [apiVersions/fileVersionsSharingToShares.feature:305](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiVersions/fileVersionsSharingToShares.feature#L305) +#### [file versions do not report the version author](https://github.com/owncloud/ocis/issues/2914) +- [apiVersions/fileVersions.feature:474](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiVersions/fileVersions.feature#L474) + #### [Getting information about a folder overwritten by a file gives 500 error instead of 404](https://github.com/owncloud/ocis/issues/1239) - [apiWebdavProperties1/copyFile.feature:226](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavProperties1/copyFile.feature#L226) - [apiWebdavProperties1/copyFile.feature:227](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavProperties1/copyFile.feature#L227) @@ -1189,8 +1192,8 @@ Not everything needs to be implemented for ocis. While the oc10 testsuite covers - [apiWebdavProperties1/copyFile.feature:108](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavProperties1/copyFile.feature#L108) - [apiWebdavProperties1/createFolder.feature:71](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavProperties1/createFolder.feature#L71) - [apiWebdavProperties1/createFolder.feature:72](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavProperties1/createFolder.feature#L72) -- [apiWebdavUpload1/uploadFile.feature:127](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavUpload1/uploadFile.feature#L127) -- [apiWebdavUpload1/uploadFile.feature:128](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavUpload1/uploadFile.feature#L128) +- [apiWebdavUpload1/uploadFile.feature:132](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavUpload1/uploadFile.feature#L132) +- [apiWebdavUpload1/uploadFile.feature:133](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavUpload1/uploadFile.feature#L133) - [apiWebdavUpload2/uploadFileToBlacklistedNameUsingOldChunking.feature:19](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavUpload2/uploadFileToBlacklistedNameUsingOldChunking.feature#L19) - [apiWebdavUpload2/uploadFileToBlacklistedNameUsingOldChunking.feature:35](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavUpload2/uploadFileToBlacklistedNameUsingOldChunking.feature#L35) - [apiWebdavUpload2/uploadFileToBlacklistedNameUsingOldChunking.feature:36](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavUpload2/uploadFileToBlacklistedNameUsingOldChunking.feature#L36) @@ -1365,3 +1368,6 @@ _ocs: api compatibility, return correct status code_ ### [Content-type is not multipart/byteranges when downloading file with Range Header](https://github.com/owncloud/ocis/issues/2677) - [apiWebdavOperations/downloadFile.feature:169](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavOperations/downloadFile.feature#L169) - [apiWebdavOperations/downloadFile.feature:170](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavOperations/downloadFile.feature#L170) + +### [Calculate personal space id in tests when running on reva](https://github.com/owncloud/core/issues/39617) +- [apiWebdavUpload1/uploadFile.feature:32](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavUpload1/uploadFile.feature#L32) From 28ad25c0366a166597896ef6179e8cdc0d3ee11d Mon Sep 17 00:00:00 2001 From: Benedikt Kulmann Date: Fri, 24 Dec 2021 13:55:10 +0100 Subject: [PATCH 27/58] Add product to ocs Version struct (#2397) The web ui will announce the backend version in the javascript console and is supposed to include the product name as well. The version seems to be a good location for the product field as it already includes the software edition as well. --- .../unreleased/enhancement-product-field-in-ocs-version.md | 5 +++++ internal/http/services/owncloud/ocs/data/capabilities.go | 1 + 2 files changed, 6 insertions(+) create mode 100644 changelog/unreleased/enhancement-product-field-in-ocs-version.md diff --git a/changelog/unreleased/enhancement-product-field-in-ocs-version.md b/changelog/unreleased/enhancement-product-field-in-ocs-version.md new file mode 100644 index 0000000000..a897200697 --- /dev/null +++ b/changelog/unreleased/enhancement-product-field-in-ocs-version.md @@ -0,0 +1,5 @@ +Enhancement: Product field in OCS version + +We've added a new field to the OCS Version, which is supposed to announce the product name. The web ui as a client will make use of it to make the backend product and version available (e.g. for easier bug reports). + +https://github.com/cs3org/reva/pull/2397 diff --git a/internal/http/services/owncloud/ocs/data/capabilities.go b/internal/http/services/owncloud/ocs/data/capabilities.go index 81ee4e3320..609f7a2979 100644 --- a/internal/http/services/owncloud/ocs/data/capabilities.go +++ b/internal/http/services/owncloud/ocs/data/capabilities.go @@ -224,4 +224,5 @@ type Version struct { Micro int `json:"micro" xml:"micro"` // = patch level String string `json:"string" xml:"string"` Edition string `json:"edition" xml:"edition"` + Product string `json:"product" xml:"product"` } From e541601fcef06cd2cf248ce80af08b45ca69b858 Mon Sep 17 00:00:00 2001 From: Saw-jan Gurung Date: Thu, 30 Dec 2021 13:48:15 +0545 Subject: [PATCH 28/58] bump core commit id for tests (#2404) --- .drone.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.env b/.drone.env index 1c52a407c5..ba6938377b 100644 --- a/.drone.env +++ b/.drone.env @@ -1,3 +1,3 @@ # The test runner source for API tests -CORE_COMMITID=fe2504387a9e914e03d6af55c54e34a617e55cf0 +CORE_COMMITID=a37efb60db92923398de7efef1abb173d13a9afb CORE_BRANCH=master From f2f4d9d3336190cc0c8b38853666f1e3bd459dd3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 5 Jan 2022 10:01:32 +0100 Subject: [PATCH 29/58] [Build-deps]: Bump github.com/mattn/go-sqlite3 from 1.14.9 to 1.14.10 (#2409) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index a0c8c8cd0f..ed76b6baaf 100644 --- a/go.mod +++ b/go.mod @@ -39,7 +39,7 @@ require ( github.com/imdario/mergo v0.3.12 // indirect github.com/jedib0t/go-pretty v4.3.0+incompatible github.com/juliangruber/go-intersect v1.1.0 - github.com/mattn/go-sqlite3 v1.14.9 + github.com/mattn/go-sqlite3 v1.14.10 github.com/mileusna/useragent v1.0.2 github.com/minio/minio-go/v7 v7.0.18 github.com/mitchellh/copystructure v1.2.0 // indirect diff --git a/go.sum b/go.sum index 73d811a43a..cad1136f98 100644 --- a/go.sum +++ b/go.sum @@ -462,8 +462,8 @@ github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzp github.com/mattn/go-runewidth v0.0.6/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= -github.com/mattn/go-sqlite3 v1.14.9 h1:10HX2Td0ocZpYEjhilsuo6WWtUqttj2Kb0KtD86/KYA= -github.com/mattn/go-sqlite3 v1.14.9/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= +github.com/mattn/go-sqlite3 v1.14.10 h1:MLn+5bFRlWMGoSRmJour3CL1w/qL96mvipqpwQW/Sfk= +github.com/mattn/go-sqlite3 v1.14.10/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/mattn/go-tty v0.0.3 h1:5OfyWorkyO7xP52Mq7tB36ajHDG5OHrmBGIS/DtakQI= github.com/mattn/go-tty v0.0.3/go.mod h1:ihxohKRERHTVzN+aSVRwACLCeqIoZAWpoICkkvrWyR0= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= From 66369ba6909687d27aac59f05e3b4b8f63aa3eba Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 5 Jan 2022 10:01:49 +0100 Subject: [PATCH 30/58] [Build-deps]: Bump github.com/minio/minio-go/v7 from 7.0.18 to 7.0.20 (#2408) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index ed76b6baaf..bc92db8847 100644 --- a/go.mod +++ b/go.mod @@ -41,7 +41,7 @@ require ( github.com/juliangruber/go-intersect v1.1.0 github.com/mattn/go-sqlite3 v1.14.10 github.com/mileusna/useragent v1.0.2 - github.com/minio/minio-go/v7 v7.0.18 + github.com/minio/minio-go/v7 v7.0.20 github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/go-testing-interface v1.14.1 // indirect github.com/mitchellh/mapstructure v1.4.3 diff --git a/go.sum b/go.sum index cad1136f98..f9befc49a4 100644 --- a/go.sum +++ b/go.sum @@ -473,8 +473,8 @@ github.com/mileusna/useragent v1.0.2 h1:DgVKtiPnjxlb73z9bCwgdUvU2nQNQ97uhgfO8l9u github.com/mileusna/useragent v1.0.2/go.mod h1:3d8TOmwL/5I8pJjyVDteHtgDGcefrFUX4ccGOMKNYYc= github.com/minio/md5-simd v1.1.0 h1:QPfiOqlZH+Cj9teu0t9b1nTBfPbyTl16Of5MeuShdK4= github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw= -github.com/minio/minio-go/v7 v7.0.18 h1:fncn6iacnK+i2uYfNc5aVPG7bEqQH0nU4yAGMSunY0w= -github.com/minio/minio-go/v7 v7.0.18/go.mod h1:SyQ1IFeJuaa+eV5yEDxW7hYE1s5VVq5sgImDe27R+zg= +github.com/minio/minio-go/v7 v7.0.20 h1:0+Xt1SkCKDgcx5cmo3UxXcJ37u5Gy+/2i/+eQYqmYJw= +github.com/minio/minio-go/v7 v7.0.20/go.mod h1:ei5JjmxwHaMrgsMrn4U/+Nmg+d8MKS1U2DAn1ou4+Do= github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU= github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= From cd9aa62fe0bb35af5d26ea9b267dd6a1a61851a1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 5 Jan 2022 10:02:11 +0100 Subject: [PATCH 31/58] [Build-deps]: Bump github.com/rs/cors from 1.8.0 to 1.8.2 (#2399) --- go.mod | 2 +- go.sum | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index bc92db8847..0c012ed49f 100644 --- a/go.mod +++ b/go.mod @@ -51,7 +51,7 @@ require ( github.com/pkg/xattr v0.4.4 github.com/pquerna/cachecontrol v0.1.0 // indirect github.com/prometheus/alertmanager v0.23.0 - github.com/rs/cors v1.8.0 + github.com/rs/cors v1.8.2 github.com/rs/zerolog v1.26.0 github.com/sciencemesh/meshdirectory-web v1.0.4 github.com/sethvargo/go-password v0.2.0 diff --git a/go.sum b/go.sum index f9befc49a4..4ff51d59b0 100644 --- a/go.sum +++ b/go.sum @@ -577,8 +577,9 @@ github.com/prometheus/statsd_exporter v0.21.0/go.mod h1:rbT83sZq2V+p73lHhPZfMc3M github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rs/cors v1.8.0 h1:P2KMzcFwrPoSjkF1WLRPsp3UMLyql8L4v9hQpVeK5so= github.com/rs/cors v1.8.0/go.mod h1:EBwu+T5AvHOcXwvZIkQFjUN6s8Czyqw12GL/Y0tUyRM= +github.com/rs/cors v1.8.2 h1:KCooALfAYGs415Cwu5ABvv9n9509fSiG5SQJn/AQo4U= +github.com/rs/cors v1.8.2/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/rs/xid v1.3.0 h1:6NjYksEUlhurdVehpc7S7dk6DAmcKv8V9gG0FsVN2U4= github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= From 0fd4b060035d38eb70f6306c3108e7810d30f989 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 5 Jan 2022 10:02:38 +0100 Subject: [PATCH 32/58] [Build-deps]: Bump github.com/ReneKroon/ttlcache/v2 (#2387) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 0c012ed49f..bf9e5cf686 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/Masterminds/goutils v1.1.1 // indirect github.com/Masterminds/semver v1.5.0 // indirect github.com/Masterminds/sprig v2.22.0+incompatible - github.com/ReneKroon/ttlcache/v2 v2.10.0 + github.com/ReneKroon/ttlcache/v2 v2.11.0 github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect github.com/aws/aws-sdk-go v1.42.19 github.com/beevik/etree v1.1.0 diff --git a/go.sum b/go.sum index 4ff51d59b0..550097fd71 100644 --- a/go.sum +++ b/go.sum @@ -52,8 +52,8 @@ github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAE github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= -github.com/ReneKroon/ttlcache/v2 v2.10.0 h1:y4g2gs2fSqAugSseq7vUB1jClqzT/e0CjSrMJywoWa8= -github.com/ReneKroon/ttlcache/v2 v2.10.0/go.mod h1:mBxvsNY+BT8qLLd6CuAJubbKo6r0jh3nb5et22bbfGY= +github.com/ReneKroon/ttlcache/v2 v2.11.0 h1:OvlcYFYi941SBN3v9dsDcC2N8vRxyHcCmJb3Vl4QMoM= +github.com/ReneKroon/ttlcache/v2 v2.11.0/go.mod h1:mBxvsNY+BT8qLLd6CuAJubbKo6r0jh3nb5et22bbfGY= github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= From 0e9cb7d5780c66f50f252368c657f4b3d12b350a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 5 Jan 2022 10:02:57 +0100 Subject: [PATCH 33/58] [Build-deps]: Bump go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc (#2359) --- go.mod | 2 +- go.sum | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index bf9e5cf686..dc2d5dd310 100644 --- a/go.mod +++ b/go.mod @@ -64,7 +64,7 @@ require ( github.com/wk8/go-ordered-map v0.2.0 go.mongodb.org/mongo-driver v1.7.2 // indirect go.opencensus.io v0.23.0 - go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.27.0 + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.28.0 go.opentelemetry.io/otel v1.3.0 go.opentelemetry.io/otel/exporters/jaeger v1.3.0 go.opentelemetry.io/otel/sdk v1.3.0 diff --git a/go.sum b/go.sum index 550097fd71..c277dc8d4f 100644 --- a/go.sum +++ b/go.sum @@ -670,16 +670,14 @@ go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.27.0 h1:TON1iU3Y5oIytGQHIejDYLam5uoSMsmA0UV9Yupb5gQ= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.27.0/go.mod h1:T/zQwBldOpoAEpE3HMbLnI8ydESZVz4ggw6Is4FF9LI= -go.opentelemetry.io/otel v1.2.0/go.mod h1:aT17Fk0Z1Nor9e0uisf98LrntPGMnk4frBO9+dkf69I= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.28.0 h1:Ky1MObd188aGbgb5OgNnwGuEEwI9MVIcc7rBW6zk5Ak= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.28.0/go.mod h1:vEhqr0m4eTc+DWxfsXoXue2GBgV2uUwVznkGIHW/e5w= go.opentelemetry.io/otel v1.3.0 h1:APxLf0eiBwLl+SOXiJJCVYzA1OOJNyAoV8C5RNRyy7Y= go.opentelemetry.io/otel v1.3.0/go.mod h1:PWIKzi6JCp7sM0k9yZ43VX+T345uNbAkDKwHVjb2PTs= go.opentelemetry.io/otel/exporters/jaeger v1.3.0 h1:HfydzioALdtcB26H5WHc4K47iTETJCdloL7VN579/L0= go.opentelemetry.io/otel/exporters/jaeger v1.3.0/go.mod h1:KoYHi1BtkUPncGSRtCe/eh1ijsnePhSkxwzz07vU0Fc= go.opentelemetry.io/otel/sdk v1.3.0 h1:3278edCoH89MEJ0Ky8WQXVmDQv3FX4ZJ3Pp+9fJreAI= go.opentelemetry.io/otel/sdk v1.3.0/go.mod h1:rIo4suHNhQwBIPg9axF8V9CA72Wz2mKF1teNrup8yzs= -go.opentelemetry.io/otel/trace v1.2.0/go.mod h1:N5FLswTubnxKxOJHM7XZC074qpeEdLy3CgAVsdMucK0= go.opentelemetry.io/otel/trace v1.3.0 h1:doy8Hzb1RJ+I3yFhtDmwNc7tIyw1tNMOIsyPzp1NOGY= go.opentelemetry.io/otel/trace v1.3.0/go.mod h1:c/VDhno8888bvQYmbYLqe41/Ldmr/KKunbvWM4/fEjk= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= From b804b9b62e67802969820e1f01996cdf09a2ed48 Mon Sep 17 00:00:00 2001 From: Willy Kloucek <34452982+wkloucek@users.noreply.github.com> Date: Wed, 5 Jan 2022 10:07:47 +0100 Subject: [PATCH 34/58] [tests-only] format .drone.star (#2411) --- .drone.star | 1616 ++++++++++++++++++++++++++------------------------- 1 file changed, 825 insertions(+), 791 deletions(-) diff --git a/.drone.star b/.drone.star index 2cff58c71f..3c60499d7d 100644 --- a/.drone.star +++ b/.drone.star @@ -1,861 +1,895 @@ # Shared step definitions def licenseScanStep(): - return { - "name": "license-scan", - "image": "registry.cern.ch/docker.io/library/golang:1.17", - "environment": { - "FOSSA_API_KEY": { - "from_secret": "fossa_api_key", - }, - }, - "detach": True, - "commands": [ - "wget -qO- https://github.com/fossas/fossa-cli/releases/download/v1.0.11/fossa-cli_1.0.11_linux_amd64.tar.gz | tar xvz -C /go/bin/", - "/go/bin/fossa analyze", - ] - } + return { + "name": "license-scan", + "image": "registry.cern.ch/docker.io/library/golang:1.17", + "environment": { + "FOSSA_API_KEY": { + "from_secret": "fossa_api_key", + }, + }, + "detach": True, + "commands": [ + "wget -qO- https://github.com/fossas/fossa-cli/releases/download/v1.0.11/fossa-cli_1.0.11_linux_amd64.tar.gz | tar xvz -C /go/bin/", + "/go/bin/fossa analyze", + ], + } def makeStep(target): - return { - "name": "build", - "image": "registry.cern.ch/docker.io/library/golang:1.17", - "commands": [ - "make %s" % target, - ] - } + return { + "name": "build", + "image": "registry.cern.ch/docker.io/library/golang:1.17", + "commands": [ + "make %s" % target, + ], + } def lintStep(): - return { - "name": "lint", - "image": "registry.cern.ch/docker.io/golangci/golangci-lint:v1.42.1", - "commands": [ - "golangci-lint run --timeout 2m0s", - ], - } + return { + "name": "lint", + "image": "registry.cern.ch/docker.io/golangci/golangci-lint:v1.42.1", + "commands": [ + "golangci-lint run --timeout 2m0s", + ], + } def cloneOc10TestReposStep(): - return { - "name": "clone-oC10-test-repos", - "image": "registry.cern.ch/docker.io/owncloudci/alpine:latest", - "commands": [ - "source /drone/src/.drone.env", - "git clone -b master --depth=1 https://github.com/owncloud/testing.git /drone/src/tmp/testing", - "git clone -b $CORE_BRANCH --single-branch --no-tags https://github.com/owncloud/core.git /drone/src/tmp/testrunner", - "cd /drone/src/tmp/testrunner", - "git checkout $CORE_COMMITID", - ], - } + return { + "name": "clone-oC10-test-repos", + "image": "registry.cern.ch/docker.io/owncloudci/alpine:latest", + "commands": [ + "source /drone/src/.drone.env", + "git clone -b master --depth=1 https://github.com/owncloud/testing.git /drone/src/tmp/testing", + "git clone -b $CORE_BRANCH --single-branch --no-tags https://github.com/owncloud/core.git /drone/src/tmp/testrunner", + "cd /drone/src/tmp/testrunner", + "git checkout $CORE_COMMITID", + ], + } # Shared service definitions def ldapService(): - return { - "name": "ldap", - "image": "registry.cern.ch/docker.io/osixia/openldap:1.3.0", - "pull": "always", - "environment": { - "LDAP_DOMAIN": "owncloud.com", - "LDAP_ORGANISATION": "ownCloud", - "LDAP_ADMIN_PASSWORD": "admin", - "LDAP_TLS_VERIFY_CLIENT": "never", - "HOSTNAME": "ldap", - }, - } + return { + "name": "ldap", + "image": "registry.cern.ch/docker.io/osixia/openldap:1.3.0", + "pull": "always", + "environment": { + "LDAP_DOMAIN": "owncloud.com", + "LDAP_ORGANISATION": "ownCloud", + "LDAP_ADMIN_PASSWORD": "admin", + "LDAP_TLS_VERIFY_CLIENT": "never", + "HOSTNAME": "ldap", + }, + } def redisService(): - return { - "name": "redis", - "image": "registry.cern.ch/docker.io/webhippie/redis", - "pull": "always", - "environment": { - "REDIS_DATABASES": 1, - }, - } + return { + "name": "redis", + "image": "registry.cern.ch/docker.io/webhippie/redis", + "pull": "always", + "environment": { + "REDIS_DATABASES": 1, + }, + } def cephService(): - return { - "name": "ceph", - "image": "ceph/daemon", - "pull": "always", - "environment": { - "CEPH_DAEMON": "demo", - "NETWORK_AUTO_DETECT": "4", - "MON_IP": "0.0.0.0", - "CEPH_PUBLIC_NETWORK": "0.0.0.0/0", - "RGW_CIVETWEB_PORT": "4000 ", - "RGW_NAME": "ceph", - "CEPH_DEMO_UID": "test-user", - "CEPH_DEMO_ACCESS_KEY": "test", - "CEPH_DEMO_SECRET_KEY": "test", - "CEPH_DEMO_BUCKET": "test", + return { + "name": "ceph", + "image": "ceph/daemon", + "pull": "always", + "environment": { + "CEPH_DAEMON": "demo", + "NETWORK_AUTO_DETECT": "4", + "MON_IP": "0.0.0.0", + "CEPH_PUBLIC_NETWORK": "0.0.0.0/0", + "RGW_CIVETWEB_PORT": "4000 ", + "RGW_NAME": "ceph", + "CEPH_DEMO_UID": "test-user", + "CEPH_DEMO_ACCESS_KEY": "test", + "CEPH_DEMO_SECRET_KEY": "test", + "CEPH_DEMO_BUCKET": "test", + }, } - } # Pipeline definitions def main(ctx): - # In order to run specific parts only, specify the parts as - # ocisIntegrationTests(6, [1, 4]) - this will only run 1st and 4th parts - # implemented for: ocisIntegrationTests and s3ngIntegrationTests - return [ - changelog(), - coverage(), - buildAndPublishDocker(), - buildOnly(), - testIntegration(), - release(), - litmusOcisOldWebdav(), - litmusOcisNewWebdav(), - litmusOcisSpacesDav(), - virtualViews(), - ] + ocisIntegrationTests(6) + s3ngIntegrationTests(12) - + # In order to run specific parts only, specify the parts as + # ocisIntegrationTests(6, [1, 4]) - this will only run 1st and 4th parts + # implemented for: ocisIntegrationTests and s3ngIntegrationTests + return [ + changelog(), + checkStarlark(), + coverage(), + buildAndPublishDocker(), + buildOnly(), + testIntegration(), + release(), + litmusOcisOldWebdav(), + litmusOcisNewWebdav(), + litmusOcisSpacesDav(), + virtualViews(), + ] + ocisIntegrationTests(6) + s3ngIntegrationTests(12) def buildAndPublishDocker(): - return { - "kind": "pipeline", - "type": "docker", - "name": "build-and-publish-docker", - "platform": { - "os": "linux", - "arch": "amd64", - }, - "trigger": { - "branch":[ - "master" - ], - "event": { - "exclude": [ - "pull_request", - "tag", - "promote", - "rollback", - ], - }, - }, - "steps": [ - { - "name": "store-dev-release", - "image": "registry.cern.ch/docker.io/library/golang:1.17", - "environment": { - "USERNAME": { - "from_secret": "cbox_username", - }, - "PASSWORD": { - "from_secret": "cbox_password", - }, + return { + "kind": "pipeline", + "type": "docker", + "name": "build-and-publish-docker", + "platform": { + "os": "linux", + "arch": "amd64", }, - "detach": True, - "commands": [ - "TZ=Europe/Berlin go run tools/create-artifacts/main.go -dev -commit ${DRONE_COMMIT} -goversion `go version | awk '{print $$3}'`", - "curl --fail -X MKCOL 'https://cernbox.cern.ch/cernbox/desktop/remote.php/webdav/eos/project/r/reva/www/daily/' -k -u $${USERNAME}:$${PASSWORD}", - "curl --fail -X MKCOL 'https://cernbox.cern.ch/cernbox/desktop/remote.php/webdav/eos/project/r/reva/www/daily/'$(date +%Y-%m-%d) -k -u $${USERNAME}:$${PASSWORD}", - "curl --fail -X MKCOL 'https://cernbox.cern.ch/cernbox/desktop/remote.php/webdav/eos/project/r/reva/www/daily/'$(date +%Y-%m-%d)'/${DRONE_COMMIT}' -k -u $${USERNAME}:$${PASSWORD}", - "for i in $(ls /drone/src/dist);do curl --fail -X PUT -u $${USERNAME}:$${PASSWORD} https://cernbox.cern.ch/cernbox/desktop/remote.php/webdav/eos/project/r/reva/www/daily/$(date +%Y-%m-%d)/${DRONE_COMMIT}/$${i} --data-binary @./dist/$${i} ; done", - ], - }, - licenseScanStep(), - makeStep("ci"), - lintStep(), - { - "name": "license-check", - "image": "registry.cern.ch/docker.io/library/golang:1.17", - "failure": "ignore", - "environment":{ - "FOSSA_API_KEY":{ - "from_secret": "fossa_api_key", - }, + "trigger": { + "branch": [ + "master", + ], + "event": { + "exclude": [ + "pull_request", + "tag", + "promote", + "rollback", + ], + }, }, - "commands": [ - "wget -qO- https://github.com/fossas/fossa-cli/releases/download/v1.0.11/fossa-cli_1.0.11_linux_amd64.tar.gz | tar xvz -C /go/bin/", - "/go/bin/fossa test --timeout 900", - ] - }, - { - "name": "publish-docker-reva-latest", - "pull": "always", - "image": "plugins/docker", - "settings": { - "repo": "cs3org/reva", - "tags": "latest", - "dockerfile": "Dockerfile.reva", - "username":{ - "from_secret": "dockerhub_username", - }, - "password": { - "from_secret": "dockerhub_password", - }, - "custom_dns":[ - "128.142.17.5", - "128.142.16.5", - ], - } - }, - { - "name": "publish-docker-revad-latest", - "pull": "always", - "image": "plugins/docker", - "settings":{ - "repo": "cs3org/revad", - "tags": "latest", - "dockerfile": "Dockerfile.revad", - "username": { - "from_secret": "dockerhub_username", - }, - "password": { - "from_secret": "dockerhub_password", - }, - "custom_dns": [ - "128.142.17.5", - "128.142.16.5", - ], - } - }, - { - "name": "publish-docker-revad-eos-latest", - "pull": "always", - "image": "plugins/docker", - "settings": { - "repo": "cs3org/revad", - "tags": "latest-eos", - "dockerfile": "Dockerfile.revad-eos", - "username": { - "from_secret": "dockerhub_username", - }, - "password": { - "from_secret": "dockerhub_password", - }, - "custom_dns": [ - "128.142.17.5", - "128.142.16.5", - ] - } - } - ] - } + "steps": [ + { + "name": "store-dev-release", + "image": "registry.cern.ch/docker.io/library/golang:1.17", + "environment": { + "USERNAME": { + "from_secret": "cbox_username", + }, + "PASSWORD": { + "from_secret": "cbox_password", + }, + }, + "detach": True, + "commands": [ + "TZ=Europe/Berlin go run tools/create-artifacts/main.go -dev -commit ${DRONE_COMMIT} -goversion `go version | awk '{print $$3}'`", + "curl --fail -X MKCOL 'https://cernbox.cern.ch/cernbox/desktop/remote.php/webdav/eos/project/r/reva/www/daily/' -k -u $${USERNAME}:$${PASSWORD}", + "curl --fail -X MKCOL 'https://cernbox.cern.ch/cernbox/desktop/remote.php/webdav/eos/project/r/reva/www/daily/'$(date +%Y-%m-%d) -k -u $${USERNAME}:$${PASSWORD}", + "curl --fail -X MKCOL 'https://cernbox.cern.ch/cernbox/desktop/remote.php/webdav/eos/project/r/reva/www/daily/'$(date +%Y-%m-%d)'/${DRONE_COMMIT}' -k -u $${USERNAME}:$${PASSWORD}", + "for i in $(ls /drone/src/dist);do curl --fail -X PUT -u $${USERNAME}:$${PASSWORD} https://cernbox.cern.ch/cernbox/desktop/remote.php/webdav/eos/project/r/reva/www/daily/$(date +%Y-%m-%d)/${DRONE_COMMIT}/$${i} --data-binary @./dist/$${i} ; done", + ], + }, + licenseScanStep(), + makeStep("ci"), + lintStep(), + { + "name": "license-check", + "image": "registry.cern.ch/docker.io/library/golang:1.17", + "failure": "ignore", + "environment": { + "FOSSA_API_KEY": { + "from_secret": "fossa_api_key", + }, + }, + "commands": [ + "wget -qO- https://github.com/fossas/fossa-cli/releases/download/v1.0.11/fossa-cli_1.0.11_linux_amd64.tar.gz | tar xvz -C /go/bin/", + "/go/bin/fossa test --timeout 900", + ], + }, + { + "name": "publish-docker-reva-latest", + "pull": "always", + "image": "plugins/docker", + "settings": { + "repo": "cs3org/reva", + "tags": "latest", + "dockerfile": "Dockerfile.reva", + "username": { + "from_secret": "dockerhub_username", + }, + "password": { + "from_secret": "dockerhub_password", + }, + "custom_dns": [ + "128.142.17.5", + "128.142.16.5", + ], + }, + }, + { + "name": "publish-docker-revad-latest", + "pull": "always", + "image": "plugins/docker", + "settings": { + "repo": "cs3org/revad", + "tags": "latest", + "dockerfile": "Dockerfile.revad", + "username": { + "from_secret": "dockerhub_username", + }, + "password": { + "from_secret": "dockerhub_password", + }, + "custom_dns": [ + "128.142.17.5", + "128.142.16.5", + ], + }, + }, + { + "name": "publish-docker-revad-eos-latest", + "pull": "always", + "image": "plugins/docker", + "settings": { + "repo": "cs3org/revad", + "tags": "latest-eos", + "dockerfile": "Dockerfile.revad-eos", + "username": { + "from_secret": "dockerhub_username", + }, + "password": { + "from_secret": "dockerhub_password", + }, + "custom_dns": [ + "128.142.17.5", + "128.142.16.5", + ], + }, + }, + ], + } def changelog(): - return { - "kind": "pipeline", - "type": "docker", - "name": "changelog", - "platform": { - "os": "linux", - "arch": "amd64", - }, - "trigger":{ - "event": { - "include": [ - "pull_request", - ], - }, - }, - "steps": [ - { - "name": "changelog", - "image": "registry.cern.ch/docker.io/library/golang:1.17", - "commands": [ - "make release-deps && /go/bin/calens > /dev/null", - "make check-changelog-drone PR=$DRONE_PULL_REQUEST", - ], - "environment": { - "GITHUB_API_TOKEN": { - "from_secret": "github_api_token", - }, - }, + return { + "kind": "pipeline", + "type": "docker", + "name": "changelog", + "platform": { + "os": "linux", + "arch": "amd64", + }, + "trigger": { + "event": { + "include": [ + "pull_request", + ], + }, + }, + "steps": [ + { + "name": "changelog", + "image": "registry.cern.ch/docker.io/library/golang:1.17", + "commands": [ + "make release-deps && /go/bin/calens > /dev/null", + "make check-changelog-drone PR=$DRONE_PULL_REQUEST", + ], + "environment": { + "GITHUB_API_TOKEN": { + "from_secret": "github_api_token", + }, + }, + }, + ], } - ] - } def coverage(): - return { - "kind": "pipeline", - "type": "docker", - "name": "unit-test-coverage", - "platform": { - "os": "linux", - "arch": "amd64", - }, - "trigger":{ - "ref":[ - "refs/heads/master", - "refs/pull/**", - ], - }, - "steps": [ - { - "name": "unit-test", - "image": "registry.cern.ch/docker.io/library/golang:1.17", - "commands": [ - "make test", - ], - }, - { - "name": "codacy", - "image": "plugins/codacy:1", - "pull": "always", - "settings": { - "token": { - "from_secret": "codacy_token", - }, + return { + "kind": "pipeline", + "type": "docker", + "name": "unit-test-coverage", + "platform": { + "os": "linux", + "arch": "amd64", + }, + "trigger": { + "ref": [ + "refs/heads/master", + "refs/pull/**", + ], }, - }, - ], - "depends_on": ['changelog'], - } + "steps": [ + { + "name": "unit-test", + "image": "registry.cern.ch/docker.io/library/golang:1.17", + "commands": [ + "make test", + ], + }, + { + "name": "codacy", + "image": "plugins/codacy:1", + "pull": "always", + "settings": { + "token": { + "from_secret": "codacy_token", + }, + }, + }, + ], + "depends_on": ["changelog"], + } def buildOnly(): - return { - "kind": "pipeline", - "type": "docker", - "name": "build-only", - "platform": { - "os": "linux", - "arch": "amd64", - }, - "trigger":{ - "event": { - "include": [ - "pull_request", - ], - }, - }, - "steps": [ - licenseScanStep(), - makeStep("ci"), - { - "name": "Docker build", - "image": "plugins/docker", - "settings": { - "repo": "n/a", - "dry_run": "true", - "dockerfile": "Dockerfile.revad", - } - }, - lintStep(), - { - "name": "license-check", - "image": "registry.cern.ch/docker.io/library/golang:1.17", - "failure": "ignore", - "environment": { - "FOSSA_API_KEY": { - "from_secret": "fossa_api_key", - }, + return { + "kind": "pipeline", + "type": "docker", + "name": "build-only", + "platform": { + "os": "linux", + "arch": "amd64", }, - "commands": [ - "wget -qO- https://github.com/fossas/fossa-cli/releases/download/v1.0.11/fossa-cli_1.0.11_linux_amd64.tar.gz | tar xvz -C /go/bin/", - "/go/bin/fossa test --timeout 900", + "trigger": { + "event": { + "include": [ + "pull_request", + ], + }, + }, + "steps": [ + licenseScanStep(), + makeStep("ci"), + { + "name": "Docker build", + "image": "plugins/docker", + "settings": { + "repo": "n/a", + "dry_run": "true", + "dockerfile": "Dockerfile.revad", + }, + }, + lintStep(), + { + "name": "license-check", + "image": "registry.cern.ch/docker.io/library/golang:1.17", + "failure": "ignore", + "environment": { + "FOSSA_API_KEY": { + "from_secret": "fossa_api_key", + }, + }, + "commands": [ + "wget -qO- https://github.com/fossas/fossa-cli/releases/download/v1.0.11/fossa-cli_1.0.11_linux_amd64.tar.gz | tar xvz -C /go/bin/", + "/go/bin/fossa test --timeout 900", + ], + }, ], - }, - ], - "depends_on": ['changelog'], - } + "depends_on": ["changelog"], + } def testIntegration(): - return { - "kind": "pipeline", - "type": "docker", - "name": "test-integration", - "platform": { - "os": "linux", - "arch": "amd64", - }, - "trigger": { - "event": { - "include": [ - "pull_request", + return { + "kind": "pipeline", + "type": "docker", + "name": "test-integration", + "platform": { + "os": "linux", + "arch": "amd64", + }, + "trigger": { + "event": { + "include": [ + "pull_request", + ], + }, + }, + "steps": [ + { + "name": "test", + "image": "registry.cern.ch/docker.io/library/golang:1.17", + "commands": [ + "make test-integration", + ], + "environment": { + "REDIS_ADDRESS": "redis:6379", + }, + }, ], - }, - }, - "steps": [ - { - "name": "test", - "image": "registry.cern.ch/docker.io/library/golang:1.17", - "commands": [ - "make test-integration", + "services": [ + redisService(), ], - "environment": { - "REDIS_ADDRESS": "redis:6379", - }, - } - ], - "services": [ - redisService(), - ], - "depends_on": ['changelog'], - } + "depends_on": ["changelog"], + } def release(): - return { - "kind": "pipeline", - "type": "docker", - "name": "release", - "platform": { - "os": "linux", - "arch": "amd64", - }, - "trigger": { - "event": { - "include": [ - "tag", - ], - }, - }, - "steps": [ - licenseScanStep(), - makeStep("ci"), - lintStep(), - { - "name": "create-dist", - "image": "registry.cern.ch/docker.io/library/golang:1.17", - "commands": [ - "go run tools/create-artifacts/main.go -version ${DRONE_TAG} -commit ${DRONE_COMMIT} -goversion `go version | awk '{print $$3}'`", - ], - }, - { - "name": "publish", - "image": "registry.cern.ch/docker.io/plugins/github-release", - "settings": { - "api_key": { - "from_secret": "github_token", - }, - "files": "dist/*", - "note": "changelog/NOTE.md", - }, - }, - { - "name": "docker-reva-tag", - "pull": "always", - "image": "plugins/docker", - "settings": { - "repo": "cs3org/reva", - "tags": "${DRONE_TAG}", - "dockerfile": "Dockerfile.reva", - "username": { - "from_secret": "dockerhub_username", - }, - "password": { - "from_secret": "dockerhub_password", - }, - "custom_dns":[ - "128.142.17.5", - "128.142.16.5", - ], - }, - }, - { - "name": "docker-revad-tag", - "pull": "always", - "image": "plugins/docker", - "settings": { - "repo": "cs3org/revad", - "tags": "${DRONE_TAG}", - "dockerfile": "Dockerfile.revad", - "username": { - "from_secret": "dockerhub_username", - }, - "password": { - "from_secret": "dockerhub_password", - }, - "custom_dns": [ - "128.142.17.5", - "128.142.16.5", - ], + return { + "kind": "pipeline", + "type": "docker", + "name": "release", + "platform": { + "os": "linux", + "arch": "amd64", }, - }, - { - "name": "docker-revad-eos-tag", - "pull": "always", - "image": "plugins/docker", - "settings": { - "repo": "cs3org/revad", - "tags": "${DRONE_TAG}-eos", - "dockerfile": "Dockerfile.revad-eos", - "username": { - "from_secret": "dockerhub_username", - }, - "password": { - "from_secret": "dockerhub_password", - }, - "custom_dns": [ - "128.142.17.5", - "128.142.16.5", - ], + "trigger": { + "event": { + "include": [ + "tag", + ], + }, }, - }, - ], - "depends_on": ['changelog'], - } + "steps": [ + licenseScanStep(), + makeStep("ci"), + lintStep(), + { + "name": "create-dist", + "image": "registry.cern.ch/docker.io/library/golang:1.17", + "commands": [ + "go run tools/create-artifacts/main.go -version ${DRONE_TAG} -commit ${DRONE_COMMIT} -goversion `go version | awk '{print $$3}'`", + ], + }, + { + "name": "publish", + "image": "registry.cern.ch/docker.io/plugins/github-release", + "settings": { + "api_key": { + "from_secret": "github_token", + }, + "files": "dist/*", + "note": "changelog/NOTE.md", + }, + }, + { + "name": "docker-reva-tag", + "pull": "always", + "image": "plugins/docker", + "settings": { + "repo": "cs3org/reva", + "tags": "${DRONE_TAG}", + "dockerfile": "Dockerfile.reva", + "username": { + "from_secret": "dockerhub_username", + }, + "password": { + "from_secret": "dockerhub_password", + }, + "custom_dns": [ + "128.142.17.5", + "128.142.16.5", + ], + }, + }, + { + "name": "docker-revad-tag", + "pull": "always", + "image": "plugins/docker", + "settings": { + "repo": "cs3org/revad", + "tags": "${DRONE_TAG}", + "dockerfile": "Dockerfile.revad", + "username": { + "from_secret": "dockerhub_username", + }, + "password": { + "from_secret": "dockerhub_password", + }, + "custom_dns": [ + "128.142.17.5", + "128.142.16.5", + ], + }, + }, + { + "name": "docker-revad-eos-tag", + "pull": "always", + "image": "plugins/docker", + "settings": { + "repo": "cs3org/revad", + "tags": "${DRONE_TAG}-eos", + "dockerfile": "Dockerfile.revad-eos", + "username": { + "from_secret": "dockerhub_username", + }, + "password": { + "from_secret": "dockerhub_password", + }, + "custom_dns": [ + "128.142.17.5", + "128.142.16.5", + ], + }, + }, + ], + "depends_on": ["changelog"], + } def virtualViews(): - return { - "kind": "pipeline", - "type": "docker", - "name": "virtual-views", - "platform": { - "os": "linux", - "arch": "amd64", - }, - "trigger": { - "event": { - "include": [ - "pull_request", - "tag", - ], - }, - }, - "steps": [ - makeStep("build-ci"), - { - "name": "revad-services", - "image": "registry.cern.ch/docker.io/library/golang:1.17", - "detach": True, - "commands": [ - "cd /drone/src/tests/oc-integration-tests/drone/", - "/drone/src/cmd/revad/revad -c frontend-global.toml &", - "/drone/src/cmd/revad/revad -c gateway.toml &", - "/drone/src/cmd/revad/revad -c storage-home-ocis.toml &", - "/drone/src/cmd/revad/revad -c storage-local-1.toml &", - "/drone/src/cmd/revad/revad -c storage-local-2.toml &", - "/drone/src/cmd/revad/revad -c users.toml", - ], - }, - cloneOc10TestReposStep(), - { - "name": "oC10APIAcceptanceTestsOcisStorage", - "image": "registry.cern.ch/docker.io/owncloudci/php:7.4", - "commands": [ - "cd /drone/src", - "make test-acceptance-api", - ], - "environment": { - "PATH_TO_CORE": "/drone/src/tmp/testrunner", - "TEST_SERVER_URL": "http://revad-services:20180", - "OCIS_REVA_DATA_ROOT": "/drone/src/tmp/reva/data/", - "DELETE_USER_DATA_CMD": "rm -rf /drone/src/tmp/reva/data/nodes/root/* /drone/src/tmp/reva/data/nodes/*-*-*-* /drone/src/tmp/reva/data/blobs/*", - "STORAGE_DRIVER": "OCIS", - "SKELETON_DIR": "/drone/src/tmp/testing/data/apiSkeleton", - "TEST_REVA": "true", - "REGULAR_USER_PASSWORD": "relativity", - "SEND_SCENARIO_LINE_REFERENCES": "true", - "BEHAT_SUITE": "apiVirtualViews", + return { + "kind": "pipeline", + "type": "docker", + "name": "virtual-views", + "platform": { + "os": "linux", + "arch": "amd64", }, - }, - ], - "depends_on": ['changelog'], - } + "trigger": { + "event": { + "include": [ + "pull_request", + "tag", + ], + }, + }, + "steps": [ + makeStep("build-ci"), + { + "name": "revad-services", + "image": "registry.cern.ch/docker.io/library/golang:1.17", + "detach": True, + "commands": [ + "cd /drone/src/tests/oc-integration-tests/drone/", + "/drone/src/cmd/revad/revad -c frontend-global.toml &", + "/drone/src/cmd/revad/revad -c gateway.toml &", + "/drone/src/cmd/revad/revad -c storage-home-ocis.toml &", + "/drone/src/cmd/revad/revad -c storage-local-1.toml &", + "/drone/src/cmd/revad/revad -c storage-local-2.toml &", + "/drone/src/cmd/revad/revad -c users.toml", + ], + }, + cloneOc10TestReposStep(), + { + "name": "oC10APIAcceptanceTestsOcisStorage", + "image": "registry.cern.ch/docker.io/owncloudci/php:7.4", + "commands": [ + "cd /drone/src", + "make test-acceptance-api", + ], + "environment": { + "PATH_TO_CORE": "/drone/src/tmp/testrunner", + "TEST_SERVER_URL": "http://revad-services:20180", + "OCIS_REVA_DATA_ROOT": "/drone/src/tmp/reva/data/", + "DELETE_USER_DATA_CMD": "rm -rf /drone/src/tmp/reva/data/nodes/root/* /drone/src/tmp/reva/data/nodes/*-*-*-* /drone/src/tmp/reva/data/blobs/*", + "STORAGE_DRIVER": "OCIS", + "SKELETON_DIR": "/drone/src/tmp/testing/data/apiSkeleton", + "TEST_REVA": "true", + "REGULAR_USER_PASSWORD": "relativity", + "SEND_SCENARIO_LINE_REFERENCES": "true", + "BEHAT_SUITE": "apiVirtualViews", + }, + }, + ], + "depends_on": ["changelog"], + } def litmusOcisOldWebdav(): - return { - "kind": "pipeline", - "type": "docker", - "name": "litmus-ocis-old-webdav", - "platform": { - "os": "linux", - "arch": "amd64", - }, - "trigger": { - "event": { - "include": [ - "pull_request", - "tag", - ], - }, - }, - "steps": [ - makeStep("build-ci"), - { - "name": "revad-services", - "image": "registry.cern.ch/docker.io/library/golang:1.17", - "detach": True, - "commands": [ - "cd /drone/src/tests/oc-integration-tests/drone/", - "/drone/src/cmd/revad/revad -c frontend.toml &", - "/drone/src/cmd/revad/revad -c gateway.toml &", - "/drone/src/cmd/revad/revad -c storage-home-ocis.toml &", - "/drone/src/cmd/revad/revad -c storage-users-ocis.toml &", - "/drone/src/cmd/revad/revad -c users.toml", - ], - }, - { - "name": "sleep-for-revad-start", - "image": "registry.cern.ch/docker.io/library/golang:1.17", - "commands": [ - "sleep 5", - ], - }, - { + return { + "kind": "pipeline", + "type": "docker", "name": "litmus-ocis-old-webdav", - "image": "registry.cern.ch/docker.io/owncloud/litmus:latest", - "environment": { - "LITMUS_URL": "http://revad-services:20080/remote.php/webdav", - "LITMUS_USERNAME": "einstein", - "LITMUS_PASSWORD": "relativity", - "TESTS": "basic http copymove props", + "platform": { + "os": "linux", + "arch": "amd64", + }, + "trigger": { + "event": { + "include": [ + "pull_request", + "tag", + ], + }, }, - }, - ], - "depends_on": ['changelog'], - } + "steps": [ + makeStep("build-ci"), + { + "name": "revad-services", + "image": "registry.cern.ch/docker.io/library/golang:1.17", + "detach": True, + "commands": [ + "cd /drone/src/tests/oc-integration-tests/drone/", + "/drone/src/cmd/revad/revad -c frontend.toml &", + "/drone/src/cmd/revad/revad -c gateway.toml &", + "/drone/src/cmd/revad/revad -c storage-home-ocis.toml &", + "/drone/src/cmd/revad/revad -c storage-users-ocis.toml &", + "/drone/src/cmd/revad/revad -c users.toml", + ], + }, + { + "name": "sleep-for-revad-start", + "image": "registry.cern.ch/docker.io/library/golang:1.17", + "commands": [ + "sleep 5", + ], + }, + { + "name": "litmus-ocis-old-webdav", + "image": "registry.cern.ch/docker.io/owncloud/litmus:latest", + "environment": { + "LITMUS_URL": "http://revad-services:20080/remote.php/webdav", + "LITMUS_USERNAME": "einstein", + "LITMUS_PASSWORD": "relativity", + "TESTS": "basic http copymove props", + }, + }, + ], + "depends_on": ["changelog"], + } def litmusOcisNewWebdav(): - return { - "kind": "pipeline", - "type": "docker", - "name": "litmus-ocis-new-webdav", - "platform": { - "os": "linux", - "arch": "amd64", - }, - "trigger": { - "event": { - "include": [ - "pull_request", - "tag", - ], - }, - }, - "steps": [ - makeStep("build-ci"), - { - "name": "revad-services", - "image": "registry.cern.ch/docker.io/library/golang:1.17", - "detach": True, - "commands": [ - "cd /drone/src/tests/oc-integration-tests/drone/", - "/drone/src/cmd/revad/revad -c frontend.toml &", - "/drone/src/cmd/revad/revad -c gateway.toml &", - "/drone/src/cmd/revad/revad -c storage-home-ocis.toml &", - "/drone/src/cmd/revad/revad -c storage-users-ocis.toml &", - "/drone/src/cmd/revad/revad -c users.toml", - ] - }, - { - "name": "sleep-for-revad-start", - "image": "registry.cern.ch/docker.io/library/golang:1.17", - "commands":[ - "sleep 5", - ], - }, - { + return { + "kind": "pipeline", + "type": "docker", "name": "litmus-ocis-new-webdav", - "image": "registry.cern.ch/docker.io/owncloud/litmus:latest", - "environment": { - # UUID is einstein user, see https://github.com/owncloud/ocis-accounts/blob/8de0530f31ed5ffb0bbb7f7f3471f87f429cb2ea/pkg/service/v0/service.go#L45 - "LITMUS_URL": "http://revad-services:20080/remote.php/dav/files/4c510ada-c86b-4815-8820-42cdf82c3d51", - "LITMUS_USERNAME": "einstein", - "LITMUS_PASSWORD": "relativity", - "TESTS": "basic http copymove props", - } - }, - ], - "depends_on": ['changelog'], - } - -def litmusOcisSpacesDav(): - return { - "kind": "pipeline", - "type": "docker", - "name": "litmus-owncloud-spaces-dav", - "platform": { - "os": "linux", - "arch": "amd64", - }, - "trigger": { - "event": { - "include": [ - "pull_request", - "tag", - ], - }, - }, - "steps": [ - makeStep("build-ci"), - { - "name": "revad-services", - "image": "registry.cern.ch/docker.io/library/golang:1.17", - "detach": True, - "commands": [ - "cd /drone/src/tests/oc-integration-tests/drone/", - "/drone/src/cmd/revad/revad -c frontend.toml &", - "/drone/src/cmd/revad/revad -c gateway.toml &", - "/drone/src/cmd/revad/revad -c storage-home-ocis.toml &", - "/drone/src/cmd/revad/revad -c storage-users-ocis.toml &", - "/drone/src/cmd/revad/revad -c users.toml", - ] - }, - { - "name": "sleep-for-revad-start", - "image": "registry.cern.ch/docker.io/library/golang:1.17", - "commands":[ - "sleep 5", - ], - }, - { - "name": "litmus-owncloud-spaces-dav", - "image": "registry.cern.ch/docker.io/owncloud/litmus:latest", - "environment": { - "LITMUS_USERNAME": "einstein", - "LITMUS_PASSWORD": "relativity", - "TESTS": "basic http copymove props", + "platform": { + "os": "linux", + "arch": "amd64", }, - "commands": [ - # The spaceid is randomly generated during the first login so we need this hack to construct the correct url. - "curl -s -k -u einstein:relativity -I http://revad-services:20080/remote.php/dav/files/einstein", - "export LITMUS_URL=http://revad-services:20080/remote.php/dav/spaces/123e4567-e89b-12d3-a456-426655440000!$(ls /drone/src/tmp/reva/data/spaces/personal/)", - "/usr/local/bin/litmus-wrapper", - ] - }, - ], - "depends_on": ['changelog'], - } - -def ocisIntegrationTests(parallelRuns, skipExceptParts = []): - pipelines = [] - debugPartsEnabled = (len(skipExceptParts) != 0) - for runPart in range(1, parallelRuns + 1): - if debugPartsEnabled and runPart not in skipExceptParts: - continue + "trigger": { + "event": { + "include": [ + "pull_request", + "tag", + ], + }, + }, + "steps": [ + makeStep("build-ci"), + { + "name": "revad-services", + "image": "registry.cern.ch/docker.io/library/golang:1.17", + "detach": True, + "commands": [ + "cd /drone/src/tests/oc-integration-tests/drone/", + "/drone/src/cmd/revad/revad -c frontend.toml &", + "/drone/src/cmd/revad/revad -c gateway.toml &", + "/drone/src/cmd/revad/revad -c storage-home-ocis.toml &", + "/drone/src/cmd/revad/revad -c storage-users-ocis.toml &", + "/drone/src/cmd/revad/revad -c users.toml", + ], + }, + { + "name": "sleep-for-revad-start", + "image": "registry.cern.ch/docker.io/library/golang:1.17", + "commands": [ + "sleep 5", + ], + }, + { + "name": "litmus-ocis-new-webdav", + "image": "registry.cern.ch/docker.io/owncloud/litmus:latest", + "environment": { + # UUID is einstein user, see https://github.com/owncloud/ocis-accounts/blob/8de0530f31ed5ffb0bbb7f7f3471f87f429cb2ea/pkg/service/v0/service.go#L45 + "LITMUS_URL": "http://revad-services:20080/remote.php/dav/files/4c510ada-c86b-4815-8820-42cdf82c3d51", + "LITMUS_USERNAME": "einstein", + "LITMUS_PASSWORD": "relativity", + "TESTS": "basic http copymove props", + }, + }, + ], + "depends_on": ["changelog"], + } - pipelines.append( - { +def litmusOcisSpacesDav(): + return { "kind": "pipeline", "type": "docker", - "name": "ocis-integration-tests-%s" % runPart, + "name": "litmus-owncloud-spaces-dav", "platform": { - "os": "linux", - "arch": "amd64", + "os": "linux", + "arch": "amd64", }, "trigger": { - "event": { - "include": [ - "pull_request", - "tag", - ], - }, + "event": { + "include": [ + "pull_request", + "tag", + ], + }, }, "steps": [ - makeStep("build-ci"), - { - "name": "revad-services", - "image": "registry.cern.ch/docker.io/library/golang:1.17", - "detach": True, - "commands": [ - "cd /drone/src/tests/oc-integration-tests/drone/", - "/drone/src/cmd/revad/revad -c frontend.toml &", - "/drone/src/cmd/revad/revad -c gateway.toml &", - "/drone/src/cmd/revad/revad -c shares.toml &", - "/drone/src/cmd/revad/revad -c storage-home-ocis.toml &", - "/drone/src/cmd/revad/revad -c storage-users-ocis.toml &", - "/drone/src/cmd/revad/revad -c storage-publiclink.toml &", - "/drone/src/cmd/revad/revad -c ldap-users.toml", - ], - }, - cloneOc10TestReposStep(), - { - "name": "oC10APIAcceptanceTestsOcisStorage", - "image": "registry.cern.ch/docker.io/owncloudci/php:7.4", - "commands": [ - "cd /drone/src/tmp/testrunner", - "make test-acceptance-api", - ], - "environment": { - "TEST_SERVER_URL": "http://revad-services:20080", - "OCIS_REVA_DATA_ROOT": "/drone/src/tmp/reva/data/", - "DELETE_USER_DATA_CMD": "rm -rf /drone/src/tmp/reva/data/nodes/root/* /drone/src/tmp/reva/data/nodes/*-*-*-* /drone/src/tmp/reva/data/blobs/*", - "STORAGE_DRIVER": "OCIS", - "SKELETON_DIR": "/drone/src/tmp/testing/data/apiSkeleton", - "TEST_WITH_LDAP": "true", - "REVA_LDAP_HOSTNAME": "ldap", - "TEST_REVA": "true", - "SEND_SCENARIO_LINE_REFERENCES": "true", - "BEHAT_FILTER_TAGS": "~@notToImplementOnOCIS&&~@toImplementOnOCIS&&~comments-app-required&&~@federation-app-required&&~@notifications-app-required&&~systemtags-app-required&&~@provisioning_api-app-required&&~@preview-extension-required&&~@local_storage&&~@skipOnOcis-OCIS-Storage&&~@skipOnOcis", - "DIVIDE_INTO_NUM_PARTS": parallelRuns, - "RUN_PART": runPart, - "EXPECTED_FAILURES_FILE": "/drone/src/tests/acceptance/expected-failures-on-OCIS-storage.md", - }, - }, - ], - "services": [ - ldapService(), + makeStep("build-ci"), + { + "name": "revad-services", + "image": "registry.cern.ch/docker.io/library/golang:1.17", + "detach": True, + "commands": [ + "cd /drone/src/tests/oc-integration-tests/drone/", + "/drone/src/cmd/revad/revad -c frontend.toml &", + "/drone/src/cmd/revad/revad -c gateway.toml &", + "/drone/src/cmd/revad/revad -c storage-home-ocis.toml &", + "/drone/src/cmd/revad/revad -c storage-users-ocis.toml &", + "/drone/src/cmd/revad/revad -c users.toml", + ], + }, + { + "name": "sleep-for-revad-start", + "image": "registry.cern.ch/docker.io/library/golang:1.17", + "commands": [ + "sleep 5", + ], + }, + { + "name": "litmus-owncloud-spaces-dav", + "image": "registry.cern.ch/docker.io/owncloud/litmus:latest", + "environment": { + "LITMUS_USERNAME": "einstein", + "LITMUS_PASSWORD": "relativity", + "TESTS": "basic http copymove props", + }, + "commands": [ + # The spaceid is randomly generated during the first login so we need this hack to construct the correct url. + "curl -s -k -u einstein:relativity -I http://revad-services:20080/remote.php/dav/files/einstein", + "export LITMUS_URL=http://revad-services:20080/remote.php/dav/spaces/123e4567-e89b-12d3-a456-426655440000!$(ls /drone/src/tmp/reva/data/spaces/personal/)", + "/usr/local/bin/litmus-wrapper", + ], + }, ], - "depends_on": ['changelog'], - } - ) + "depends_on": ["changelog"], + } - return pipelines +def ocisIntegrationTests(parallelRuns, skipExceptParts = []): + pipelines = [] + debugPartsEnabled = (len(skipExceptParts) != 0) + for runPart in range(1, parallelRuns + 1): + if debugPartsEnabled and runPart not in skipExceptParts: + continue + + pipelines.append( + { + "kind": "pipeline", + "type": "docker", + "name": "ocis-integration-tests-%s" % runPart, + "platform": { + "os": "linux", + "arch": "amd64", + }, + "trigger": { + "event": { + "include": [ + "pull_request", + "tag", + ], + }, + }, + "steps": [ + makeStep("build-ci"), + { + "name": "revad-services", + "image": "registry.cern.ch/docker.io/library/golang:1.17", + "detach": True, + "commands": [ + "cd /drone/src/tests/oc-integration-tests/drone/", + "/drone/src/cmd/revad/revad -c frontend.toml &", + "/drone/src/cmd/revad/revad -c gateway.toml &", + "/drone/src/cmd/revad/revad -c shares.toml &", + "/drone/src/cmd/revad/revad -c storage-home-ocis.toml &", + "/drone/src/cmd/revad/revad -c storage-users-ocis.toml &", + "/drone/src/cmd/revad/revad -c storage-publiclink.toml &", + "/drone/src/cmd/revad/revad -c ldap-users.toml", + ], + }, + cloneOc10TestReposStep(), + { + "name": "oC10APIAcceptanceTestsOcisStorage", + "image": "registry.cern.ch/docker.io/owncloudci/php:7.4", + "commands": [ + "cd /drone/src/tmp/testrunner", + "make test-acceptance-api", + ], + "environment": { + "TEST_SERVER_URL": "http://revad-services:20080", + "OCIS_REVA_DATA_ROOT": "/drone/src/tmp/reva/data/", + "DELETE_USER_DATA_CMD": "rm -rf /drone/src/tmp/reva/data/nodes/root/* /drone/src/tmp/reva/data/nodes/*-*-*-* /drone/src/tmp/reva/data/blobs/*", + "STORAGE_DRIVER": "OCIS", + "SKELETON_DIR": "/drone/src/tmp/testing/data/apiSkeleton", + "TEST_WITH_LDAP": "true", + "REVA_LDAP_HOSTNAME": "ldap", + "TEST_REVA": "true", + "SEND_SCENARIO_LINE_REFERENCES": "true", + "BEHAT_FILTER_TAGS": "~@notToImplementOnOCIS&&~@toImplementOnOCIS&&~comments-app-required&&~@federation-app-required&&~@notifications-app-required&&~systemtags-app-required&&~@provisioning_api-app-required&&~@preview-extension-required&&~@local_storage&&~@skipOnOcis-OCIS-Storage&&~@skipOnOcis", + "DIVIDE_INTO_NUM_PARTS": parallelRuns, + "RUN_PART": runPart, + "EXPECTED_FAILURES_FILE": "/drone/src/tests/acceptance/expected-failures-on-OCIS-storage.md", + }, + }, + ], + "services": [ + ldapService(), + ], + "depends_on": ["changelog"], + }, + ) + + return pipelines def s3ngIntegrationTests(parallelRuns, skipExceptParts = []): - pipelines = [] - debugPartsEnabled = (len(skipExceptParts) != 0) - for runPart in range(1, parallelRuns + 1): - if debugPartsEnabled and runPart not in skipExceptParts: - continue + pipelines = [] + debugPartsEnabled = (len(skipExceptParts) != 0) + for runPart in range(1, parallelRuns + 1): + if debugPartsEnabled and runPart not in skipExceptParts: + continue + + pipelines.append( + { + "kind": "pipeline", + "type": "docker", + "name": "s3ng-integration-tests-%s" % runPart, + "platform": { + "os": "linux", + "arch": "amd64", + }, + "trigger": { + "event": { + "include": [ + "pull_request", + "tag", + ], + }, + }, + "steps": [ + makeStep("build-ci"), + { + "name": "revad-services", + "image": "registry.cern.ch/docker.io/library/golang:1.17", + "detach": True, + "commands": [ + "cd /drone/src/tests/oc-integration-tests/drone/", + "/drone/src/cmd/revad/revad -c frontend.toml &", + "/drone/src/cmd/revad/revad -c gateway.toml &", + "/drone/src/cmd/revad/revad -c shares.toml &", + "/drone/src/cmd/revad/revad -c storage-home-s3ng.toml &", + "/drone/src/cmd/revad/revad -c storage-users-s3ng.toml &", + "/drone/src/cmd/revad/revad -c storage-publiclink.toml &", + "/drone/src/cmd/revad/revad -c ldap-users.toml", + ], + }, + cloneOc10TestReposStep(), + { + "name": "oC10APIAcceptanceTestsS3ngStorage", + "image": "registry.cern.ch/docker.io/owncloudci/php:7.4", + "commands": [ + "cd /drone/src/tmp/testrunner", + "make test-acceptance-api", + ], + "environment": { + "TEST_SERVER_URL": "http://revad-services:20080", + "OCIS_REVA_DATA_ROOT": "/drone/src/tmp/reva/data/", + "DELETE_USER_DATA_CMD": "rm -rf /drone/src/tmp/reva/data/nodes/root/* /drone/src/tmp/reva/data/nodes/*-*-*-* /drone/src/tmp/reva/data/blobs/*", + "STORAGE_DRIVER": "S3NG", + "SKELETON_DIR": "/drone/src/tmp/testing/data/apiSkeleton", + "TEST_WITH_LDAP": "true", + "REVA_LDAP_HOSTNAME": "ldap", + "TEST_REVA": "true", + "SEND_SCENARIO_LINE_REFERENCES": "true", + "BEHAT_FILTER_TAGS": "~@notToImplementOnOCIS&&~@toImplementOnOCIS&&~comments-app-required&&~@federation-app-required&&~@notifications-app-required&&~systemtags-app-required&&~@provisioning_api-app-required&&~@preview-extension-required&&~@local_storage&&~@skipOnOcis-OCIS-Storage&&~@skipOnOcis", + "DIVIDE_INTO_NUM_PARTS": parallelRuns, + "RUN_PART": runPart, + "EXPECTED_FAILURES_FILE": "/drone/src/tests/acceptance/expected-failures-on-S3NG-storage.md", + }, + }, + ], + "services": [ + ldapService(), + cephService(), + ], + "depends_on": ["changelog"], + }, + ) - pipelines.append( - { + return pipelines + +def checkStarlark(): + return { "kind": "pipeline", "type": "docker", - "name": "s3ng-integration-tests-%s" % runPart, - "platform": { - "os": "linux", - "arch": "amd64", - }, - "trigger": { - "event": { - "include": [ - "pull_request", - "tag", - ], - }, - }, + "name": "check-starlark", "steps": [ - makeStep("build-ci"), - { - "name": "revad-services", - "image": "registry.cern.ch/docker.io/library/golang:1.17", - "detach": True, - "commands": [ - "cd /drone/src/tests/oc-integration-tests/drone/", - "/drone/src/cmd/revad/revad -c frontend.toml &", - "/drone/src/cmd/revad/revad -c gateway.toml &", - "/drone/src/cmd/revad/revad -c shares.toml &", - "/drone/src/cmd/revad/revad -c storage-home-s3ng.toml &", - "/drone/src/cmd/revad/revad -c storage-users-s3ng.toml &", - "/drone/src/cmd/revad/revad -c storage-publiclink.toml &", - "/drone/src/cmd/revad/revad -c ldap-users.toml", - ], - }, - cloneOc10TestReposStep(), - { - "name": "oC10APIAcceptanceTestsS3ngStorage", - "image": "registry.cern.ch/docker.io/owncloudci/php:7.4", - "commands": [ - "cd /drone/src/tmp/testrunner", - "make test-acceptance-api", - ], - "environment": { - "TEST_SERVER_URL": "http://revad-services:20080", - "OCIS_REVA_DATA_ROOT": "/drone/src/tmp/reva/data/", - "DELETE_USER_DATA_CMD": "rm -rf /drone/src/tmp/reva/data/nodes/root/* /drone/src/tmp/reva/data/nodes/*-*-*-* /drone/src/tmp/reva/data/blobs/*", - "STORAGE_DRIVER": "S3NG", - "SKELETON_DIR": "/drone/src/tmp/testing/data/apiSkeleton", - "TEST_WITH_LDAP": "true", - "REVA_LDAP_HOSTNAME": "ldap", - "TEST_REVA": "true", - "SEND_SCENARIO_LINE_REFERENCES": "true", - "BEHAT_FILTER_TAGS": "~@notToImplementOnOCIS&&~@toImplementOnOCIS&&~comments-app-required&&~@federation-app-required&&~@notifications-app-required&&~systemtags-app-required&&~@provisioning_api-app-required&&~@preview-extension-required&&~@local_storage&&~@skipOnOcis-OCIS-Storage&&~@skipOnOcis", - "DIVIDE_INTO_NUM_PARTS": parallelRuns, - "RUN_PART": runPart, - "EXPECTED_FAILURES_FILE": "/drone/src/tests/acceptance/expected-failures-on-S3NG-storage.md", - + { + "name": "format-check-starlark", + "image": "registry.cern.ch/docker.io/owncloudci/bazel-buildifier:latest", + "commands": [ + "buildifier --mode=check .drone.star", + ], + }, + { + "name": "show-diff", + "image": "registry.cern.ch/docker.io/owncloudci/bazel-buildifier:latest", + "commands": [ + "buildifier --mode=fix .drone.star", + "git diff", + ], + "when": { + "status": [ + "failure", + ], + }, }, - }, - ], - "services": [ - ldapService(), - cephService(), ], - "depends_on": ['changelog'], - } - ) - - return pipelines + "depends_on": [], + "trigger": { + "ref": [ + "refs/pull/**", + ], + }, + } From 2f21d4d0120f91804965cf8321f1ffbabf7e7c9a Mon Sep 17 00:00:00 2001 From: Willy Kloucek <34452982+wkloucek@users.noreply.github.com> Date: Wed, 5 Jan 2022 10:11:13 +0100 Subject: [PATCH 35/58] update tus/tusd to version 1.8.0 (#2393) --- .../unreleased/enhancement-update-tusd.md | 6 + go.mod | 8 +- go.sum | 162 ++++++++++++++++-- pkg/storage/fs/owncloudsql/upload.go | 4 + pkg/storage/utils/decomposedfs/upload.go | 4 + pkg/storage/utils/localfs/upload.go | 4 + 6 files changed, 171 insertions(+), 17 deletions(-) create mode 100644 changelog/unreleased/enhancement-update-tusd.md diff --git a/changelog/unreleased/enhancement-update-tusd.md b/changelog/unreleased/enhancement-update-tusd.md new file mode 100644 index 0000000000..d4b6deabdd --- /dev/null +++ b/changelog/unreleased/enhancement-update-tusd.md @@ -0,0 +1,6 @@ +Enhancement: update tus/tusd to version 1.8.0 + +We've update tus/tusd to version 1.8.0. + +https://github.com/cs3org/reva/issues/2393 +https://github.com/cs3org/reva/pull/2224 diff --git a/go.mod b/go.mod index dc2d5dd310..794ce3ae57 100644 --- a/go.mod +++ b/go.mod @@ -60,7 +60,7 @@ require ( github.com/studio-b12/gowebdav v0.0.0-20210917133250-a3a86976a1df github.com/thanhpk/randstr v1.0.4 github.com/tidwall/pretty v1.2.0 // indirect - github.com/tus/tusd v1.6.0 + github.com/tus/tusd v1.8.0 github.com/wk8/go-ordered-map v0.2.0 go.mongodb.org/mongo-driver v1.7.2 // indirect go.opencensus.io v0.23.0 @@ -70,10 +70,10 @@ require ( go.opentelemetry.io/otel/sdk v1.3.0 go.opentelemetry.io/otel/trace v1.3.0 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 - golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f - golang.org/x/sys v0.0.0-20210921065528-437939a70204 + golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1 + golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359 golang.org/x/term v0.0.0-20210916214954-140adaaadfaf - google.golang.org/genproto v0.0.0-20200825200019-8632dd797987 + google.golang.org/genproto v0.0.0-20211021150943-2b146023228c google.golang.org/grpc v1.42.0 google.golang.org/protobuf v1.27.1 gopkg.in/square/go-jose.v2 v2.6.0 // indirect diff --git a/go.sum b/go.sum index c277dc8d4f..5d9dec820b 100644 --- a/go.sum +++ b/go.sum @@ -14,8 +14,20 @@ cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bP cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= -cloud.google.com/go v0.65.0 h1:Dg9iHVQfrhq82rUNu9ZxUDrJLaxFUe/HlCVaLyRruq8= cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= +cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= +cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= +cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= +cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= +cloud.google.com/go v0.83.0/go.mod h1:Z7MJUsANfY0pYPdw0lbnivPx4/vhy/e2FEkSkF7vAVY= +cloud.google.com/go v0.84.0/go.mod h1:RazrYuxIK6Kb7YrzzhPoLmCVzl7Sup4NrbKPg8KHSUM= +cloud.google.com/go v0.87.0/go.mod h1:TpDYlFy7vuLzZMMZ+B6iRiELaY7z/gJPaqbMx6mlWcY= +cloud.google.com/go v0.90.0/go.mod h1:kRX0mNRHe0e2rC6oNakvwQqzyDmg57xJ+SZU1eT2aDQ= +cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+YI= +cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4= +cloud.google.com/go v0.97.0 h1:3DXvAyifywvq64LfkKaMOmkWPS1CikIQdMe2lY9vxU8= +cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= @@ -33,9 +45,18 @@ cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0Zeo cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +cloud.google.com/go/storage v1.18.2/go.mod h1:AiIj7BWXyhO5gGVmYJ+S8tbkCx3yb0IMjua8Aw4naVM= contrib.go.opencensus.io/exporter/prometheus v0.4.0 h1:0QfIkj9z/iVZgK31D9H9ohjjIDApI2GOPScCKwxedbs= contrib.go.opencensus.io/exporter/prometheus v0.4.0/go.mod h1:o7cosnyfuPVK0tB8q0QmaQNhGnptITnPQB+z1+qeFB0= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/Azure/azure-pipeline-go v0.2.3/go.mod h1:x841ezTBIMG6O3lAcl8ATHnsOPVl2bqk7S3ta6S6u4k= +github.com/Azure/azure-storage-blob-go v0.14.0/go.mod h1:SMqIBi+SuiQH32bvyjngEewEeXoPfKMgWlBDaYf6fck= +github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= +github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= +github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= +github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= +github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= +github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/Azure/go-ntlmssp v0.0.0-20200615164410-66371956d46c h1:/IBSNwUN8+eKzUzbJPqhK839ygXJ82sde8x3ogr6R28= github.com/Azure/go-ntlmssp v0.0.0-20200615164410-66371956d46c/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= @@ -77,6 +98,7 @@ github.com/aws/aws-sdk-go v1.20.1/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN github.com/aws/aws-sdk-go v1.34.28/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48= github.com/aws/aws-sdk-go v1.38.35/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/aws/aws-sdk-go v1.40.11/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q= +github.com/aws/aws-sdk-go v1.41.13/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q= github.com/aws/aws-sdk-go v1.42.19 h1:L/aM1QwsqVia9qIqexTHwYN+lgLYuOtf11VDgz0YIyw= github.com/aws/aws-sdk-go v1.42.19/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q= github.com/beevik/etree v1.1.0 h1:T0xke/WvNtMoCqgzPhkX2r4rjY3GDZFi+FjpRZY2Jbs= @@ -127,6 +149,7 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= +github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= @@ -297,8 +320,10 @@ github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFU github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.4 h1:l75CXGRSwbaYNpl/Z2X1XIIAMSCquvXgpVZDhwEIJsc= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= +github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= +github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -314,10 +339,12 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/gomodule/redigo v1.8.6 h1:h7kHSqUl2kxeaQtVslsfUCPJ1oz2pxcyzLy4zezIzPw= github.com/gomodule/redigo v1.8.6/go.mod h1:P9dn9mFrCBvWhGE1wpxx6fgq7BAeLBk+UUUzlpkBYO0= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -342,6 +369,8 @@ github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= @@ -349,13 +378,23 @@ github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= +github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/grpc-ecosystem/go-grpc-middleware v1.1.0/go.mod h1:f5nM7jw/oeRSadq3xCzHAvxcr8HZnzsqU6ILg/0NiiE= @@ -384,6 +423,7 @@ github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpO github.com/huandu/xstrings v1.3.2 h1:L18LIDzqlW6xN2rEkpdV8+oL/IXWJ1APd+vsdYy4Wdw= github.com/huandu/xstrings v1.3.2/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= @@ -430,8 +470,9 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxv github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= @@ -451,6 +492,7 @@ github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaO github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.7 h1:bQGKb3vps/j0E9GfJQ03JyhRuxsvdAanXlT9BTw3mdw= github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-ieproxy v0.0.1/go.mod h1:pYabZ6IHcRpFh7vIaLfK7rdcWgFEb3SFJ6/gNWuh88E= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= @@ -506,7 +548,6 @@ github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uYLpLIr5fm8diHn0JbqRycJi6w0Ms= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= @@ -632,12 +673,13 @@ github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhV github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tus/tusd v1.1.0/go.mod h1:3DWPOdeCnjBwKtv98y5dSws3itPqfce5TVa0s59LRiA= -github.com/tus/tusd v1.6.0 h1:IU9Z2Z5FZfHIap6NPFbPItyx/eU6aN87z4ya/mPzS4g= -github.com/tus/tusd v1.6.0/go.mod h1:ygrT4B9ZSb27dx3uTnobX5nOFDnutBL6iWKLH4+KpA0= +github.com/tus/tusd v1.8.0 h1:QODQ5uMhL2tFX3Ouk7rUHHqPqeDBvi2+gYIoyUO0n8Q= +github.com/tus/tusd v1.8.0/go.mod h1:stZzKpol4qz7lX2HXy/1H526dn5mRnkIICTW2lrh9NM= github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw= github.com/vimeo/go-util v1.2.0/go.mod h1:s13SMDTSO7AjH1nbgp707mfN5JFIWUFDU5MDDuRRtKs= +github.com/vimeo/go-util v1.4.1/go.mod h1:r+yspV//C48HeMXV8nEvtUeNiIiGfVv3bbEHzOgudwE= github.com/wk8/go-ordered-map v0.2.0 h1:KlvGyHstD1kkGZkPtHCyCfRYS0cz84uk6rrW/Dnhdtk= github.com/wk8/go-ordered-map v0.2.0/go.mod h1:9ZIbRunKbuvfPKyBP1SIKLcXNlv74YCOZ3t3VTS6gRk= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= @@ -668,6 +710,7 @@ go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.28.0 h1:Ky1MObd188aGbgb5OgNnwGuEEwI9MVIcc7rBW6zk5Ak= @@ -700,6 +743,7 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 h1:7I4JAnoQBe7ZtJcBaYHi5UtiO8tQHbUSXxL+pnGRANg= @@ -725,8 +769,9 @@ golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5 h1:2M3HP5CCK1Si9FQhwnzYhXdG6DXeebvUHFpre8QvbyI= golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhpJuEoBZuwh1m2j7U6Iug= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= @@ -735,6 +780,8 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180530234432-1e491301e022/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -755,6 +802,7 @@ golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191112182307-2180aed22343/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -772,13 +820,17 @@ golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= +golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= @@ -789,9 +841,18 @@ golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4Iltr golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f h1:Qmd2pbz05z7z6lm0DrgQVVPuBm92jqujBKMHMOlOQEw= +golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1 h1:B333XXssMuKQeBwiNODx4TupZy7bf4sxFZnN2ZOcvUE= +golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -833,6 +894,7 @@ golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191112214154-59a1497f0cea/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -853,23 +915,39 @@ golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200828194041-157a740278f4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200918174421-af09f7315aff/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201101102859-da207088b7d1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210921065528-437939a70204 h1:JJhkWtBuTQKyz2bd5WG9H8iUsJRU3En/KRfN8B2RnDs= -golang.org/x/sys v0.0.0-20210921065528-437939a70204/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210917161153-d61c044b1678/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359 h1:2B5p2L5IfGiD7+b9BOoRMC6DgObAVZV+Fsp050NqXik= +golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210916214954-140adaaadfaf h1:Ihq/mm/suC88gF8WFcVwk+OV6Tq+wyA1O0E5UEvDglI= @@ -938,9 +1016,19 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= +golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210112230658-8b4aab62c064/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.7 h1:6j8CgantCy3yc8JGBqkDLMKWqZ0RDU2g1HVgacojGWQ= golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= @@ -966,12 +1054,28 @@ google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0M google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= +google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= +google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= +google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= +google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= +google.golang.org/api v0.47.0/go.mod h1:Wbvgpq1HddcWVtzsVLyfLp8lDg6AA241LmgIL59tHXo= +google.golang.org/api v0.48.0/go.mod h1:71Pr1vy+TAZRPkPs/xlCf5SsU8WjuAWv1Pfjbtukyy4= +google.golang.org/api v0.50.0/go.mod h1:4bNT5pAuq5ji4SRZm+5QIkjny9JAyVD/3gaSihNefaw= +google.golang.org/api v0.51.0/go.mod h1:t4HdrdoNgyN5cbEfm7Lum0lcLDLiise1F8qDKX00sOU= +google.golang.org/api v0.54.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6z3k= +google.golang.org/api v0.55.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= +google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= +google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdrMgI= +google.golang.org/api v0.58.0/go.mod h1:cAbP2FsxoGVNwtgNAmmn3y5G1TWAiVYRmg4yku3lv+E= +google.golang.org/api v0.60.0/go.mod h1:d7rl65NZAkEQ90JFzqBjcRq1TVeG5ZoGV3sSpEnnVb4= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.6 h1:lMO5rYAqUxkmaj76jAkRUvt5JZgFymx/+Q5Mzfivuhc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20170818010345-ee236bd376b0/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190404172233-64821d5d2107/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -1003,10 +1107,40 @@ google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEY google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200825200019-8632dd797987 h1:PDIOdWxZ8eRizhKa1AAvY53xsvLB1cWorMjslvY3VA8= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= +google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= +google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210604141403-392c879c8b08/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210608205507-b6d2f5bf0d7d/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= +google.golang.org/genproto v0.0.0-20210713002101-d411969a0d9a/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= +google.golang.org/genproto v0.0.0-20210716133855-ce7ef5c701ea/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= +google.golang.org/genproto v0.0.0-20210728212813-7823e685a01f/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= +google.golang.org/genproto v0.0.0-20210805201207-89edb61ffb67/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= +google.golang.org/genproto v0.0.0-20210813162853-db860fec028c/go.mod h1:cFeNkxwySK631ADgubI+/XFU/xp8FD5KIVV4rj8UC5w= +google.golang.org/genproto v0.0.0-20210821163610-241b8fcbd6c8/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210917145530-b395a37504d4/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211016002631-37fc39342514/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211021150943-2b146023228c h1:FqrtZMB5Wr+/RecOM3uPJNPfWR8Upb5hAPnt7PU6i4k= +google.golang.org/genproto v0.0.0-20211021150943-2b146023228c/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/grpc v1.26.0 h1:2dTRdpdFEEhJYQD8EMLB61nnrzSCTbG38PhqdhvOltg= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -1026,13 +1160,15 @@ gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLks gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= gopkg.in/go-playground/validator.v9 v9.29.1/go.mod h1:+c9/zcJMFNgbLvly1L1V+PpxWdVbfP1avr/N00E2vyQ= gopkg.in/h2non/gock.v1 v1.0.14/go.mod h1:sX4zAkdYX1TRGJ2JY156cFspQn4yRWn6p9EMdODlynE= +gopkg.in/h2non/gock.v1 v1.1.2/go.mod h1:n7UGz/ckNChHiK05rDoiC4MYSunEC/lyaUm2WWaDva0= gopkg.in/ini.v1 v1.57.0 h1:9unxIsFcTt4I55uWluz+UmL95q4kdJ0buvQ1ZIqVQww= gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/square/go-jose.v2 v2.6.0 h1:NGk74WTnPKBNUhNzQX7PYcTLUjoq7mzKk2OKbvwk2iI= diff --git a/pkg/storage/fs/owncloudsql/upload.go b/pkg/storage/fs/owncloudsql/upload.go index 3ebb2aeae3..da167efe91 100644 --- a/pkg/storage/fs/owncloudsql/upload.go +++ b/pkg/storage/fs/owncloudsql/upload.go @@ -251,6 +251,10 @@ func (fs *owncloudsqlfs) GetUpload(ctx context.Context, id string) (tusd.Upload, info := tusd.FileInfo{} data, err := ioutil.ReadFile(infoPath) if err != nil { + if os.IsNotExist(err) { + // Interpret os.ErrNotExist as 404 Not Found + err = tusd.ErrNotFound + } return nil, err } if err := json.Unmarshal(data, &info); err != nil { diff --git a/pkg/storage/utils/decomposedfs/upload.go b/pkg/storage/utils/decomposedfs/upload.go index 2ed872e6a1..4d1f2b9c0f 100644 --- a/pkg/storage/utils/decomposedfs/upload.go +++ b/pkg/storage/utils/decomposedfs/upload.go @@ -306,6 +306,10 @@ func (fs *Decomposedfs) GetUpload(ctx context.Context, id string) (tusd.Upload, info := tusd.FileInfo{} data, err := ioutil.ReadFile(infoPath) if err != nil { + if os.IsNotExist(err) { + // Interpret os.ErrNotExist as 404 Not Found + err = tusd.ErrNotFound + } return nil, err } if err := json.Unmarshal(data, &info); err != nil { diff --git a/pkg/storage/utils/localfs/upload.go b/pkg/storage/utils/localfs/upload.go index fb91a6b358..36963e7bfb 100644 --- a/pkg/storage/utils/localfs/upload.go +++ b/pkg/storage/utils/localfs/upload.go @@ -213,6 +213,10 @@ func (fs *localfs) GetUpload(ctx context.Context, id string) (tusd.Upload, error info := tusd.FileInfo{} data, err := ioutil.ReadFile(infoPath) if err != nil { + if os.IsNotExist(err) { + // Interpret os.ErrNotExist as 404 Not Found + err = tusd.ErrNotFound + } return nil, err } if err := json.Unmarshal(data, &info); err != nil { From f953398b0ab82de28cf925d4c7e4088f968b0fcf Mon Sep 17 00:00:00 2001 From: Ishank Arora Date: Wed, 5 Jan 2022 10:11:44 +0100 Subject: [PATCH 36/58] Fixes for apps in public shares, project spaces for EOS driver (#2371) --- changelog/unreleased/eos-fixes.md | 3 ++ .../publicstorageprovider.go | 6 ++- internal/http/interceptors/auth/auth.go | 15 ++++--- pkg/app/provider/wopi/wopi.go | 14 +++++- pkg/auth/manager/publicshares/publicshares.go | 14 +++++- pkg/auth/scope/publicshare.go | 10 ++++- pkg/auth/scope/resourceinfo.go | 10 ++++- pkg/cbox/favorite/sql/sql.go | 2 +- pkg/cbox/storage/eoswrapper/eoswrapper.go | 1 + pkg/cbox/utils/conversions.go | 15 ++++++- pkg/eosclient/eosbinary/eosbinary.go | 12 +++-- pkg/eosclient/eosclient.go | 2 +- pkg/eosclient/eosgrpc/eosgrpc.go | 3 +- pkg/storage/utils/eosfs/eosfs.go | 45 +++++++++++++++++-- pkg/storage/utils/grants/grants.go | 10 ++--- pkg/storage/utils/localfs/localfs.go | 2 +- 16 files changed, 128 insertions(+), 36 deletions(-) create mode 100644 changelog/unreleased/eos-fixes.md diff --git a/changelog/unreleased/eos-fixes.md b/changelog/unreleased/eos-fixes.md new file mode 100644 index 0000000000..e0d0cbac7a --- /dev/null +++ b/changelog/unreleased/eos-fixes.md @@ -0,0 +1,3 @@ +Bugfix: Fixes for apps in public shares, project spaces for EOS driver + +https://github.com/cs3org/reva/pull/2370 \ No newline at end of file diff --git a/internal/grpc/services/publicstorageprovider/publicstorageprovider.go b/internal/grpc/services/publicstorageprovider/publicstorageprovider.go index 9aa3aed324..9bdc72057c 100644 --- a/internal/grpc/services/publicstorageprovider/publicstorageprovider.go +++ b/internal/grpc/services/publicstorageprovider/publicstorageprovider.go @@ -596,8 +596,10 @@ func (s *service) augmentStatResponse(ctx context.Context, res *provider.StatRes } func (s *service) setPublicStorageID(info *provider.ResourceInfo, shareToken string) { - info.Id.StorageId = s.mountID - info.Id.OpaqueId = shareToken + "/" + info.Id.OpaqueId + if s.mountID != "" { + info.Id.StorageId = s.mountID + info.Id.OpaqueId = shareToken + "/" + info.Id.OpaqueId + } } func addShare(i *provider.ResourceInfo, ls *link.PublicShare) error { diff --git a/internal/http/interceptors/auth/auth.go b/internal/http/interceptors/auth/auth.go index 42988a0973..4e5a07c80e 100644 --- a/internal/http/interceptors/auth/auth.go +++ b/internal/http/interceptors/auth/auth.go @@ -21,6 +21,7 @@ package auth import ( "fmt" "net/http" + "strings" "time" "github.com/bluele/gcache" @@ -297,14 +298,16 @@ func getCredsForUserAgent(ua string, uam map[string]string, creds []string) []st return creds } - cred, ok := uam[ua] - if ok { - for _, v := range creds { - if v == cred { - return []string{cred} + for u, cred := range uam { + if strings.Contains(ua, u) { + for _, v := range creds { + if v == cred { + return []string{cred} + } } + return creds + } - return creds } return creds diff --git a/pkg/app/provider/wopi/wopi.go b/pkg/app/provider/wopi/wopi.go index 77e49817e4..5f2a38add1 100644 --- a/pkg/app/provider/wopi/wopi.go +++ b/pkg/app/provider/wopi/wopi.go @@ -136,10 +136,20 @@ func (p *wopiProvider) GetAppURL(ctx context.Context, resource *provider.Resourc q.Add("fileid", resource.GetId().OpaqueId) q.Add("endpoint", resource.GetId().StorageId) q.Add("viewmode", viewMode.String()) + u, ok := ctxpkg.ContextGetUser(ctx) if ok { // else defaults to "Guest xyz" - q.Add("username", u.Username) - q.Add("userid", u.Id.OpaqueId+"@"+u.Id.Idp) + var isPublicShare bool + if u.Opaque != nil { + if _, ok := u.Opaque.Map["public-share-role"]; ok { + isPublicShare = true + } + } + + if !isPublicShare { + q.Add("username", u.Username) + q.Add("userid", u.Id.OpaqueId+"@"+u.Id.Idp) + } } q.Add("appname", p.conf.AppName) diff --git a/pkg/auth/manager/publicshares/publicshares.go b/pkg/auth/manager/publicshares/publicshares.go index c00f37f996..0dc4de28be 100644 --- a/pkg/auth/manager/publicshares/publicshares.go +++ b/pkg/auth/manager/publicshares/publicshares.go @@ -136,15 +136,27 @@ func (m *manager) Authenticate(ctx context.Context, token, secret string) (*user share := publicShareResponse.GetShare() role := authpb.Role_ROLE_VIEWER + roleStr := "viewer" if share.Permissions.Permissions.InitiateFileUpload { role = authpb.Role_ROLE_EDITOR + roleStr = "editor" } scope, err := scope.AddPublicShareScope(share, role, nil) if err != nil { return nil, nil, err } - return getUserResponse.GetUser(), scope, nil + u := getUserResponse.GetUser() + u.Opaque = &types.Opaque{ + Map: map[string]*types.OpaqueEntry{ + "public-share-role": { + Decoder: "plain", + Value: []byte(roleStr), + }, + }, + } + + return u, scope, nil } // ErrPasswordNotProvided is returned when the public share is password protected, but there was no password on the request diff --git a/pkg/auth/scope/publicshare.go b/pkg/auth/scope/publicshare.go index 7de007ca87..bfa538040f 100644 --- a/pkg/auth/scope/publicshare.go +++ b/pkg/auth/scope/publicshare.go @@ -23,8 +23,10 @@ import ( "fmt" "strings" + appprovider "github.com/cs3org/go-cs3apis/cs3/app/provider/v1beta1" appregistry "github.com/cs3org/go-cs3apis/cs3/app/registry/v1beta1" authpb "github.com/cs3org/go-cs3apis/cs3/auth/provider/v1beta1" + gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1" userv1beta1 "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" link "github.com/cs3org/go-cs3apis/cs3/sharing/link/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" @@ -52,9 +54,13 @@ func publicshareScope(ctx context.Context, scope *authpb.Scope, resource interfa return checkStorageRef(ctx, &share, v.GetRef()), nil case *provider.InitiateFileDownloadRequest: return checkStorageRef(ctx, &share, v.GetRef()), nil + case *appprovider.OpenInAppRequest: + return checkStorageRef(ctx, &share, &provider.Reference{ResourceId: v.ResourceInfo.Id}), nil + case *gateway.OpenInAppRequest: + return checkStorageRef(ctx, &share, v.GetRef()), nil - // Editor role - // need to return appropriate status codes in the ocs/ocdav layers. + // Editor role + // need to return appropriate status codes in the ocs/ocdav layers. case *provider.CreateContainerRequest: return hasRoleEditor(*scope) && checkStorageRef(ctx, &share, v.GetRef()), nil case *provider.TouchFileRequest: diff --git a/pkg/auth/scope/resourceinfo.go b/pkg/auth/scope/resourceinfo.go index d2ad4240fc..2066ce5667 100644 --- a/pkg/auth/scope/resourceinfo.go +++ b/pkg/auth/scope/resourceinfo.go @@ -23,7 +23,9 @@ import ( "fmt" "strings" + appprovider "github.com/cs3org/go-cs3apis/cs3/app/provider/v1beta1" authpb "github.com/cs3org/go-cs3apis/cs3/auth/provider/v1beta1" + gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" registry "github.com/cs3org/go-cs3apis/cs3/storage/registry/v1beta1" "github.com/rs/zerolog" @@ -50,9 +52,13 @@ func resourceinfoScope(_ context.Context, scope *authpb.Scope, resource interfac return checkResourceInfo(&r, v.GetRef()), nil case *provider.InitiateFileDownloadRequest: return checkResourceInfo(&r, v.GetRef()), nil + case *appprovider.OpenInAppRequest: + return checkResourceInfo(&r, &provider.Reference{ResourceId: v.ResourceInfo.Id}), nil + case *gateway.OpenInAppRequest: + return checkResourceInfo(&r, v.GetRef()), nil - // Editor role - // need to return appropriate status codes in the ocs/ocdav layers. + // Editor role + // need to return appropriate status codes in the ocs/ocdav layers. case *provider.CreateContainerRequest: return hasRoleEditor(*scope) && checkResourceInfo(&r, v.GetRef()), nil case *provider.TouchFileRequest: diff --git a/pkg/cbox/favorite/sql/sql.go b/pkg/cbox/favorite/sql/sql.go index 3a49b8c99c..ebb6c85ec9 100644 --- a/pkg/cbox/favorite/sql/sql.go +++ b/pkg/cbox/favorite/sql/sql.go @@ -97,7 +97,7 @@ func (m *mgr) SetFavorite(ctx context.Context, userID *user.UserId, resourceInfo // The primary key is just the ID in the table, it should ideally be (uid, fileid_prefix, fileid, tag_key) // For the time being, just check if the favorite already exists. If it does, return early var id int - query := `"SELECT id FROM cbox_metadata WHERE uid=? AND fileid_prefix=? AND fileid=? AND tag_key="fav"` + query := `SELECT id FROM cbox_metadata WHERE uid=? AND fileid_prefix=? AND fileid=? AND tag_key="fav"` if err := m.db.QueryRow(query, user.Id.OpaqueId, resourceInfo.Id.StorageId, resourceInfo.Id.OpaqueId).Scan(&id); err == nil { // Favorite is already set, return return nil diff --git a/pkg/cbox/storage/eoswrapper/eoswrapper.go b/pkg/cbox/storage/eoswrapper/eoswrapper.go index bc55aa30c7..bc4467ba5d 100644 --- a/pkg/cbox/storage/eoswrapper/eoswrapper.go +++ b/pkg/cbox/storage/eoswrapper/eoswrapper.go @@ -166,6 +166,7 @@ func (w *wrapper) setProjectSharingPermissions(ctx context.Context, r *provider. r.PermissionSet.RemoveGrant = true r.PermissionSet.UpdateGrant = true r.PermissionSet.ListGrants = true + r.PermissionSet.GetQuota = true return nil } } diff --git a/pkg/cbox/utils/conversions.go b/pkg/cbox/utils/conversions.go index 787880ec89..1391df675a 100644 --- a/pkg/cbox/utils/conversions.go +++ b/pkg/cbox/utils/conversions.go @@ -113,9 +113,12 @@ func ResourceTypeToItemInt(r provider.ResourceType) int { // SharePermToInt maps read/write permissions to an integer func SharePermToInt(p *provider.ResourcePermissions) int { var perm int - if p.CreateContainer || p.InitiateFileUpload { + switch { + case p.InitiateFileUpload && !p.InitiateFileDownload: + perm = 4 + case p.InitiateFileUpload: perm = 15 - } else if p.ListContainer || p.InitiateFileDownload { + case p.InitiateFileDownload: perm = 1 } // TODO map denials and resharing; currently, denials are mapped to 0 @@ -158,6 +161,14 @@ func IntTosharePerm(p int, itemType string) *provider.ResourcePermissions { perm.PurgeRecycle = true } return perm + case 4: + return &provider.ResourcePermissions{ + Stat: true, + ListContainer: true, + GetPath: true, + CreateContainer: true, + InitiateFileUpload: true, + } default: // TODO we may have other options, for now this is a denial return &provider.ResourcePermissions{} diff --git a/pkg/eosclient/eosbinary/eosbinary.go b/pkg/eosclient/eosbinary/eosbinary.go index b9c1f6a5a4..91b45dd037 100644 --- a/pkg/eosclient/eosbinary/eosbinary.go +++ b/pkg/eosclient/eosbinary/eosbinary.go @@ -535,12 +535,16 @@ func (c *Client) SetAttr(ctx context.Context, auth eosclient.Authorization, attr } // UnsetAttr unsets an extended attribute on a path. -func (c *Client) UnsetAttr(ctx context.Context, auth eosclient.Authorization, attr *eosclient.Attribute, path string) error { +func (c *Client) UnsetAttr(ctx context.Context, auth eosclient.Authorization, attr *eosclient.Attribute, recursive bool, path string) error { if !isValidAttribute(attr) { return errors.New("eos: attr is invalid: " + serializeAttribute(attr)) } - - args := []string{"attr", "-r", "rm", fmt.Sprintf("%d.%s", attr.Type, attr.Key), path} + var args []string + if recursive { + args = []string{"attr", "-r", "rm", fmt.Sprintf("%s.%s", attrTypeToString(attr.Type), attr.Key), path} + } else { + args = []string{"attr", "rm", fmt.Sprintf("%s.%s", attrTypeToString(attr.Type), attr.Key), path} + } _, _, err := c.executeEOS(ctx, args, auth) if err != nil { return err @@ -911,7 +915,7 @@ func (c *Client) parseFind(ctx context.Context, auth eosclient.Authorization, di for _, fi := range finfos { // For files, inherit ACLs from the parent // And set the inode to that of their version folder - if !fi.IsDir { + if !fi.IsDir && !isVersionFolder(dirPath) { if parent != nil { fi.SysACL.Entries = append(fi.SysACL.Entries, parent.SysACL.Entries...) } diff --git a/pkg/eosclient/eosclient.go b/pkg/eosclient/eosclient.go index c4e47e7b23..5e52854c78 100644 --- a/pkg/eosclient/eosclient.go +++ b/pkg/eosclient/eosclient.go @@ -36,7 +36,7 @@ type EOSClient interface { GetFileInfoByFXID(ctx context.Context, auth Authorization, fxid string) (*FileInfo, error) GetFileInfoByPath(ctx context.Context, auth Authorization, path string) (*FileInfo, error) SetAttr(ctx context.Context, auth Authorization, attr *Attribute, recursive bool, path string) error - UnsetAttr(ctx context.Context, auth Authorization, attr *Attribute, path string) error + UnsetAttr(ctx context.Context, auth Authorization, attr *Attribute, recursive bool, path string) error GetAttr(ctx context.Context, auth Authorization, key, path string) (*Attribute, error) GetQuota(ctx context.Context, username string, rootAuth Authorization, path string) (*QuotaInfo, error) SetQuota(ctx context.Context, rooAuth Authorization, info *SetQuotaInfo) error diff --git a/pkg/eosclient/eosgrpc/eosgrpc.go b/pkg/eosclient/eosgrpc/eosgrpc.go index ea906c9de5..8cd4c62b6a 100644 --- a/pkg/eosclient/eosgrpc/eosgrpc.go +++ b/pkg/eosclient/eosgrpc/eosgrpc.go @@ -572,7 +572,7 @@ func (c *Client) SetAttr(ctx context.Context, auth eosclient.Authorization, attr } // UnsetAttr unsets an extended attribute on a path. -func (c *Client) UnsetAttr(ctx context.Context, auth eosclient.Authorization, attr *eosclient.Attribute, path string) error { +func (c *Client) UnsetAttr(ctx context.Context, auth eosclient.Authorization, attr *eosclient.Attribute, recursive bool, path string) error { log := appctx.GetLogger(ctx) log.Info().Str("func", "UnsetAttr").Str("uid,gid", auth.Role.UID+","+auth.Role.GID).Str("path", path).Msg("") @@ -586,6 +586,7 @@ func (c *Client) UnsetAttr(ctx context.Context, auth eosclient.Authorization, at var ktd = []string{attr.Key} msg.Keystodelete = ktd + msg.Recursive = recursive msg.Id = new(erpc.MDId) msg.Id.Path = []byte(path) diff --git a/pkg/storage/utils/eosfs/eosfs.go b/pkg/storage/utils/eosfs/eosfs.go index 89cd109cdc..028e7d90d7 100644 --- a/pkg/storage/utils/eosfs/eosfs.go +++ b/pkg/storage/utils/eosfs/eosfs.go @@ -526,7 +526,7 @@ func (fs *eosfs) UnsetArbitraryMetadata(ctx context.Context, ref *provider.Refer Key: k, } - err := fs.c.UnsetAttr(ctx, auth, attr, fn) + err := fs.c.UnsetAttr(ctx, auth, attr, false, fn) if err != nil { return errors.Wrap(err, "eosfs: error unsetting xattr in eos driver") } @@ -796,7 +796,7 @@ func (fs *eosfs) ListGrants(ctx context.Context, ref *provider.Reference) ([]*pr grantList = append(grantList, &provider.Grant{ Grantee: grantee, - Permissions: grants.GetGrantPermissionSet(a.Permissions, true), + Permissions: grants.GetGrantPermissionSet(a.Permissions), }) } @@ -1670,7 +1670,7 @@ func (fs *eosfs) convertToFileReference(ctx context.Context, eosFileInfo *eoscli return nil, err } info.Type = provider.ResourceType_RESOURCE_TYPE_REFERENCE - val, ok := eosFileInfo.Attrs["user.reva.target"] + val, ok := eosFileInfo.Attrs["reva.target"] if !ok || val == "" { return nil, errtypes.InternalError("eosfs: reference does not contain target: target=" + val + " file=" + eosFileInfo.File) } @@ -1688,6 +1688,43 @@ func (fs *eosfs) permissionSet(ctx context.Context, eosFileInfo *eosclient.FileI } if owner != nil && u.Id.OpaqueId == owner.OpaqueId && u.Id.Idp == owner.Idp { + // The logged-in user is the owner but we may be impersonating them + // on behalf of a public share accessor. + + if u.Opaque != nil { + if publicShare, ok := u.Opaque.Map["public-share-role"]; ok { + if string(publicShare.Value) == "editor" { + return &provider.ResourcePermissions{ + CreateContainer: true, + Delete: true, + GetPath: true, + GetQuota: true, + InitiateFileDownload: true, + InitiateFileUpload: true, + ListContainer: true, + ListFileVersions: true, + ListGrants: true, + ListRecycle: true, + Move: true, + PurgeRecycle: true, + RestoreFileVersion: true, + RestoreRecycleItem: true, + Stat: true, + } + } + return &provider.ResourcePermissions{ + GetPath: true, + GetQuota: true, + InitiateFileDownload: true, + ListContainer: true, + ListFileVersions: true, + ListRecycle: true, + ListGrants: true, + Stat: true, + } + } + } + return &provider.ResourcePermissions{ // owner has all permissions AddGrant: true, @@ -1738,7 +1775,7 @@ func (fs *eosfs) permissionSet(ctx context.Context, eosFileInfo *eosclient.FileI } if (e.Type == acl.TypeUser && e.Qualifier == auth.Role.UID) || (e.Type == acl.TypeLightweight && e.Qualifier == u.Id.OpaqueId) || userInGroup { - mergePermissions(&perm, grants.GetGrantPermissionSet(e.Permissions, eosFileInfo.IsDir)) + mergePermissions(&perm, grants.GetGrantPermissionSet(e.Permissions)) } } diff --git a/pkg/storage/utils/grants/grants.go b/pkg/storage/utils/grants/grants.go index 57efbdb8da..72dbdc31db 100644 --- a/pkg/storage/utils/grants/grants.go +++ b/pkg/storage/utils/grants/grants.go @@ -66,7 +66,7 @@ func GetACLPerm(set *provider.ResourcePermissions) (string, error) { // TODO(labkode): add more fine grained controls. // EOS acls are a mix of ACLs and POSIX permissions. More details can be found in // https://github.com/cern-eos/eos/blob/master/doc/configuration/permission.rst -func GetGrantPermissionSet(perm string, isDir bool) *provider.ResourcePermissions { +func GetGrantPermissionSet(perm string) *provider.ResourcePermissions { var rp provider.ResourcePermissions // default to 0 == all denied if strings.Contains(perm, "r") && !strings.Contains(perm, "!r") { @@ -82,17 +82,13 @@ func GetGrantPermissionSet(perm string, isDir bool) *provider.ResourcePermission rp.InitiateFileUpload = true rp.RestoreFileVersion = true rp.RestoreRecycleItem = true - if isDir { - rp.CreateContainer = true - } + rp.CreateContainer = true } if strings.Contains(perm, "x") && !strings.Contains(perm, "!x") { rp.ListFileVersions = true rp.ListRecycle = true - if isDir { - rp.ListContainer = true - } + rp.ListContainer = true } if strings.Contains(perm, "!d") { diff --git a/pkg/storage/utils/localfs/localfs.go b/pkg/storage/utils/localfs/localfs.go index af70f404e3..aa55d27715 100644 --- a/pkg/storage/utils/localfs/localfs.go +++ b/pkg/storage/utils/localfs/localfs.go @@ -497,7 +497,7 @@ func (fs *localfs) ListGrants(ctx context.Context, ref *provider.Reference) ([]* } else if grantSplit[0] == acl.TypeGroup { grantee.Id = &provider.Grantee_GroupId{GroupId: &grouppb.GroupId{OpaqueId: parts[0], Idp: parts[1]}} } - permissions := grants.GetGrantPermissionSet(role, true) + permissions := grants.GetGrantPermissionSet(role) grantList = append(grantList, &provider.Grant{ Grantee: grantee, From 0f9142a96d5fca08df4dfa1c304f9ff561755a87 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 5 Jan 2022 10:59:29 +0100 Subject: [PATCH 37/58] [Build-deps]: Bump github.com/aws/aws-sdk-go from 1.42.19 to 1.42.27 (#2414) --- go.mod | 2 +- go.sum | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 794ce3ae57..17b80d0774 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/Masterminds/sprig v2.22.0+incompatible github.com/ReneKroon/ttlcache/v2 v2.11.0 github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect - github.com/aws/aws-sdk-go v1.42.19 + github.com/aws/aws-sdk-go v1.42.27 github.com/beevik/etree v1.1.0 github.com/bluele/gcache v0.0.2 github.com/c-bata/go-prompt v0.2.5 diff --git a/go.sum b/go.sum index 5d9dec820b..1eb61ae856 100644 --- a/go.sum +++ b/go.sum @@ -99,8 +99,8 @@ github.com/aws/aws-sdk-go v1.34.28/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/ github.com/aws/aws-sdk-go v1.38.35/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/aws/aws-sdk-go v1.40.11/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q= github.com/aws/aws-sdk-go v1.41.13/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q= -github.com/aws/aws-sdk-go v1.42.19 h1:L/aM1QwsqVia9qIqexTHwYN+lgLYuOtf11VDgz0YIyw= -github.com/aws/aws-sdk-go v1.42.19/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q= +github.com/aws/aws-sdk-go v1.42.27 h1:kxsBXQg3ee6LLbqjp5/oUeDgG7TENFrWYDmEVnd7spU= +github.com/aws/aws-sdk-go v1.42.27/go.mod h1:OGr6lGMAKGlG9CVrYnWYDKIyb829c6EVBRjxqjmPepc= github.com/beevik/etree v1.1.0 h1:T0xke/WvNtMoCqgzPhkX2r4rjY3GDZFi+FjpRZY2Jbs= github.com/beevik/etree v1.1.0/go.mod h1:r8Aw8JqVegEf0w2fDnATrX9VpkMcyFeM0FhwO62wh+A= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= @@ -834,8 +834,9 @@ golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d h1:20cMwl2fHAzkJMEA+8J4JgqBQcQGzbisXo31MIeenXI= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211216030914-fe4d6282115f h1:hEYJvxw1lSnWIl8X9ofsYMklzaDs90JI2az5YMd4fPM= +golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= From fa1d1fc01d859c2711622675ecc28d3f9992a656 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 5 Jan 2022 10:59:55 +0100 Subject: [PATCH 38/58] [Build-deps]: Bump github.com/rs/zerolog from 1.26.0 to 1.26.1 (#2388) --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 17b80d0774..b4ad8f6ecd 100644 --- a/go.mod +++ b/go.mod @@ -52,7 +52,7 @@ require ( github.com/pquerna/cachecontrol v0.1.0 // indirect github.com/prometheus/alertmanager v0.23.0 github.com/rs/cors v1.8.2 - github.com/rs/zerolog v1.26.0 + github.com/rs/zerolog v1.26.1 github.com/sciencemesh/meshdirectory-web v1.0.4 github.com/sethvargo/go-password v0.2.0 github.com/stretchr/objx v0.3.0 // indirect @@ -69,7 +69,7 @@ require ( go.opentelemetry.io/otel/exporters/jaeger v1.3.0 go.opentelemetry.io/otel/sdk v1.3.0 go.opentelemetry.io/otel/trace v1.3.0 - golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 + golang.org/x/crypto v0.0.0-20211215165025-cf75a172585e golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1 golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359 golang.org/x/term v0.0.0-20210916214954-140adaaadfaf diff --git a/go.sum b/go.sum index 1eb61ae856..fc9a547584 100644 --- a/go.sum +++ b/go.sum @@ -624,8 +624,8 @@ github.com/rs/cors v1.8.2/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/rs/xid v1.3.0 h1:6NjYksEUlhurdVehpc7S7dk6DAmcKv8V9gG0FsVN2U4= github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= -github.com/rs/zerolog v1.26.0 h1:ORM4ibhEZeTeQlCojCK2kPz1ogAY4bGs4tD+SaAdGaE= -github.com/rs/zerolog v1.26.0/go.mod h1:yBiM87lvSqX8h0Ww4sdzNSkVYZ8dL2xjZJG1lAuGZEo= +github.com/rs/zerolog v1.26.1 h1:/ihwxqH+4z8UxyI70wM1z9yCvkWcfz/a3mj48k/Zngc= +github.com/rs/zerolog v1.26.1/go.mod h1:/wSSJWX7lVrsOwlbyTRSOJvqRlc+WjWlfes+CiJ+tmc= github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/sciencemesh/meshdirectory-web v1.0.4 h1:1YSctF6PAXhoHUYCaeRTj7rHaF7b3rYrZf2R0VXBIbo= github.com/sciencemesh/meshdirectory-web v1.0.4/go.mod h1:fJSThTS3xf+sTdL0iXQoaQJssLI7tn7DetHMHUl4SRk= @@ -746,8 +746,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 h1:7I4JAnoQBe7ZtJcBaYHi5UtiO8tQHbUSXxL+pnGRANg= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20211215165025-cf75a172585e h1:1SzTfNOXwIS2oWiMF+6qu0OUDKb0dauo6MoDUQyu+yU= +golang.org/x/crypto v0.0.0-20211215165025-cf75a172585e/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= From cedc008d6b50526f2868fcee49b11d4aff2f4815 Mon Sep 17 00:00:00 2001 From: PKiran <39373750+kiranparajuli589@users.noreply.github.com> Date: Fri, 7 Jan 2022 12:15:47 +0545 Subject: [PATCH 39/58] update owncloud core commit id (#2418) --- .drone.env | 2 +- tests/acceptance/expected-failures-on-OCIS-storage.md | 8 +++++++- tests/acceptance/expected-failures-on-S3NG-storage.md | 8 +++++++- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/.drone.env b/.drone.env index ba6938377b..86f199a58b 100644 --- a/.drone.env +++ b/.drone.env @@ -1,3 +1,3 @@ # The test runner source for API tests -CORE_COMMITID=a37efb60db92923398de7efef1abb173d13a9afb +CORE_COMMITID=09d584745d6cbd6aebc557d9b78f6130e9b99e2b CORE_BRANCH=master diff --git a/tests/acceptance/expected-failures-on-OCIS-storage.md b/tests/acceptance/expected-failures-on-OCIS-storage.md index ab20e85f30..d3eed0341e 100644 --- a/tests/acceptance/expected-failures-on-OCIS-storage.md +++ b/tests/acceptance/expected-failures-on-OCIS-storage.md @@ -1343,7 +1343,13 @@ _ocs: api compatibility, return correct status code_ - [apiVersions/fileVersionsSharingToShares.feature:305](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiVersions/fileVersionsSharingToShares.feature#L305) #### [file versions do not report the version author](https://github.com/owncloud/ocis/issues/2914) -- [apiVersions/fileVersions.feature:474](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiVersions/fileVersions.feature#L474) +- [apiVersions/fileVersionAuthor.feature:14](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiVersions/fileVersionAuthor.feature#L14) +- [apiVersions/fileVersionAuthor.feature:36](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiVersions/fileVersionAuthor.feature#L36) +- [apiVersions/fileVersionAuthor.feature:56](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiVersions/fileVersionAuthor.feature#L56) +- [apiVersions/fileVersionAuthor.feature:75](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiVersions/fileVersionAuthor.feature#L75) +- [apiVersions/fileVersionAuthor.feature:101](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiVersions/fileVersionAuthor.feature#L101) +- [apiVersions/fileVersionAuthor.feature:128](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiVersions/fileVersionAuthor.feature#L128) +- [apiVersions/fileVersionAuthor.feature:155](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiVersions/fileVersionAuthor.feature#L155) ### [Allow public link sharing only for certain groups feature not implemented] - [apiSharePublicLink2/allowGroupToCreatePublicLinks.feature:35](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiSharePublicLink2/allowGroupToCreatePublicLinks.feature#L35) diff --git a/tests/acceptance/expected-failures-on-S3NG-storage.md b/tests/acceptance/expected-failures-on-S3NG-storage.md index 684c92e08d..6d8ee7cf79 100644 --- a/tests/acceptance/expected-failures-on-S3NG-storage.md +++ b/tests/acceptance/expected-failures-on-S3NG-storage.md @@ -17,7 +17,13 @@ Basic file management like up and download, move, copy, properties, quota, trash - [apiVersions/fileVersionsSharingToShares.feature:305](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiVersions/fileVersionsSharingToShares.feature#L305) #### [file versions do not report the version author](https://github.com/owncloud/ocis/issues/2914) -- [apiVersions/fileVersions.feature:474](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiVersions/fileVersions.feature#L474) +- [apiVersions/fileVersionAuthor.feature:14](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiVersions/fileVersionAuthor.feature#L14) +- [apiVersions/fileVersionAuthor.feature:36](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiVersions/fileVersionAuthor.feature#L36) +- [apiVersions/fileVersionAuthor.feature:56](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiVersions/fileVersionAuthor.feature#L56) +- [apiVersions/fileVersionAuthor.feature:75](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiVersions/fileVersionAuthor.feature#L75) +- [apiVersions/fileVersionAuthor.feature:101](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiVersions/fileVersionAuthor.feature#L101) +- [apiVersions/fileVersionAuthor.feature:128](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiVersions/fileVersionAuthor.feature#L128) +- [apiVersions/fileVersionAuthor.feature:155](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiVersions/fileVersionAuthor.feature#L155) #### [Getting information about a folder overwritten by a file gives 500 error instead of 404](https://github.com/owncloud/ocis/issues/1239) - [apiWebdavProperties1/copyFile.feature:226](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavProperties1/copyFile.feature#L226) From b563723af198f62860c46a3ccb5b7d300369248c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 Jan 2022 11:09:50 +0100 Subject: [PATCH 40/58] [Build-deps]: Bump github.com/mattn/go-sqlite3 (#2425) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index b4ad8f6ecd..7bf5859a58 100644 --- a/go.mod +++ b/go.mod @@ -39,7 +39,7 @@ require ( github.com/imdario/mergo v0.3.12 // indirect github.com/jedib0t/go-pretty v4.3.0+incompatible github.com/juliangruber/go-intersect v1.1.0 - github.com/mattn/go-sqlite3 v1.14.10 + github.com/mattn/go-sqlite3 v2.0.3+incompatible github.com/mileusna/useragent v1.0.2 github.com/minio/minio-go/v7 v7.0.20 github.com/mitchellh/copystructure v1.2.0 // indirect diff --git a/go.sum b/go.sum index fc9a547584..a37d8b8ca2 100644 --- a/go.sum +++ b/go.sum @@ -504,8 +504,8 @@ github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzp github.com/mattn/go-runewidth v0.0.6/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= -github.com/mattn/go-sqlite3 v1.14.10 h1:MLn+5bFRlWMGoSRmJour3CL1w/qL96mvipqpwQW/Sfk= -github.com/mattn/go-sqlite3 v1.14.10/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= +github.com/mattn/go-sqlite3 v2.0.3+incompatible h1:gXHsfypPkaMZrKbD5209QV9jbUTJKjyR5WD3HYQSd+U= +github.com/mattn/go-sqlite3 v2.0.3+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/go-tty v0.0.3 h1:5OfyWorkyO7xP52Mq7tB36ajHDG5OHrmBGIS/DtakQI= github.com/mattn/go-tty v0.0.3/go.mod h1:ihxohKRERHTVzN+aSVRwACLCeqIoZAWpoICkkvrWyR0= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= From 3ac52f9ec37586907a43397569d97ce9ea73a55f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 Jan 2022 11:10:02 +0100 Subject: [PATCH 41/58] [Build-deps]: Bump github.com/gomodule/redigo from 1.8.6 to 1.8.8 (#2426) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 7bf5859a58..abb2defa94 100644 --- a/go.mod +++ b/go.mod @@ -27,7 +27,7 @@ require ( github.com/go-sql-driver/mysql v1.6.0 github.com/golang-jwt/jwt v3.2.2+incompatible github.com/golang/protobuf v1.5.2 - github.com/gomodule/redigo v1.8.6 + github.com/gomodule/redigo v1.8.8 github.com/google/go-cmp v0.5.6 github.com/google/go-github v17.0.0+incompatible github.com/google/go-querystring v1.1.0 // indirect diff --git a/go.sum b/go.sum index a37d8b8ca2..b2612f2e5c 100644 --- a/go.sum +++ b/go.sum @@ -345,8 +345,8 @@ github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiu github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/gomodule/redigo v1.8.6 h1:h7kHSqUl2kxeaQtVslsfUCPJ1oz2pxcyzLy4zezIzPw= -github.com/gomodule/redigo v1.8.6/go.mod h1:P9dn9mFrCBvWhGE1wpxx6fgq7BAeLBk+UUUzlpkBYO0= +github.com/gomodule/redigo v1.8.8 h1:f6cXq6RRfiyrOJEV7p3JhLDlmawGBVBBP1MggY8Mo4E= +github.com/gomodule/redigo v1.8.8/go.mod h1:7ArFNvsTjH8GMMzB4uy1snslv2BwmginuMs06a1uzZE= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= From 32b439e51b946d0d1089f5bcb21135fad59b0574 Mon Sep 17 00:00:00 2001 From: Ishank Arora Date: Wed, 12 Jan 2022 11:34:21 +0100 Subject: [PATCH 42/58] OIDC and WOPI changes for lightweight users (#2278) --- changelog/unreleased/oidc-lw-users.md | 3 ++ internal/grpc/interceptors/auth/scope.go | 3 ++ .../grpc/services/gateway/authprovider.go | 15 +++--- .../services/gateway/usershareprovider.go | 4 +- pkg/app/provider/wopi/wopi.go | 7 ++- pkg/auth/manager/oidc/oidc.go | 23 ++++++++- pkg/auth/scope/lightweight.go | 32 ++++++++++++- pkg/auth/scope/receivedshare.go | 5 +- pkg/auth/scope/share.go | 5 +- pkg/cbox/user/rest/rest.go | 47 ++++++++++++++++--- pkg/eosclient/eosbinary/eosbinary.go | 2 +- pkg/storage/utils/acl/acl.go | 26 +++++++++- pkg/storage/utils/eosfs/eosfs.go | 15 +++++- 13 files changed, 164 insertions(+), 23 deletions(-) create mode 100644 changelog/unreleased/oidc-lw-users.md diff --git a/changelog/unreleased/oidc-lw-users.md b/changelog/unreleased/oidc-lw-users.md new file mode 100644 index 0000000000..2053f69551 --- /dev/null +++ b/changelog/unreleased/oidc-lw-users.md @@ -0,0 +1,3 @@ +Enhancement: OIDC driver changes for lightweight users + +https://github.com/cs3org/reva/pull/2278 \ No newline at end of file diff --git a/internal/grpc/interceptors/auth/scope.go b/internal/grpc/interceptors/auth/scope.go index 8bad148478..d2161b6eb5 100644 --- a/internal/grpc/interceptors/auth/scope.go +++ b/internal/grpc/interceptors/auth/scope.go @@ -120,6 +120,9 @@ func expandAndVerifyScope(ctx context.Context, req interface{}, tokenScope map[s if utils.ResourceIDEqual(share.Share.ResourceId, ref.GetResourceId()) { return nil } + if ok, err := checkIfNestedResource(ctx, ref, share.Share.ResourceId, client, mgr); err == nil && ok { + return nil + } } } else if strings.HasPrefix(k, "publicshare") { var share link.PublicShare diff --git a/internal/grpc/services/gateway/authprovider.go b/internal/grpc/services/gateway/authprovider.go index 81156facd8..e8092f0b5f 100644 --- a/internal/grpc/services/gateway/authprovider.go +++ b/internal/grpc/services/gateway/authprovider.go @@ -21,7 +21,6 @@ package gateway import ( "context" "fmt" - "strings" authpb "github.com/cs3org/go-cs3apis/cs3/auth/provider/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/auth/provider/v1beta1" @@ -29,17 +28,13 @@ import ( gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1" userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" - collaboration "github.com/cs3org/go-cs3apis/cs3/sharing/collaboration/v1beta1" - link "github.com/cs3org/go-cs3apis/cs3/sharing/link/v1beta1" storageprovider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/cs3org/reva/pkg/appctx" - "github.com/cs3org/reva/pkg/auth/scope" ctxpkg "github.com/cs3org/reva/pkg/ctx" "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/rgrpc/status" "github.com/cs3org/reva/pkg/rgrpc/todo/pool" "github.com/cs3org/reva/pkg/sharedconf" - "github.com/cs3org/reva/pkg/utils" "github.com/pkg/errors" "google.golang.org/grpc/metadata" ) @@ -121,13 +116,19 @@ func (s *svc) Authenticate(ctx context.Context, req *gateway.AuthenticateRequest ctx = ctxpkg.ContextSetToken(ctx, token) ctx = ctxpkg.ContextSetUser(ctx, res.User) ctx = metadata.AppendToOutgoingContext(ctx, ctxpkg.TokenHeader, token) - scope, err := s.expandScopes(ctx, res.TokenScope) + + // Commenting out as the token size can get too big + // For now, we'll try to resolve all resources on every request + // TODO(ishank011): Add a cache for these + /* scope, err := s.expandScopes(ctx, res.TokenScope) if err != nil { err = errors.Wrap(err, "authsvc: error expanding token scope") return &gateway.AuthenticateResponse{ Status: status.NewUnauthenticated(ctx, err, "error expanding access token scope"), }, nil } + */ + scope := res.TokenScope token, err = s.tokenmgr.MintToken(ctx, &u, scope) if err != nil { @@ -241,6 +242,7 @@ func (s *svc) findAuthProvider(ctx context.Context, authType string) (provider.P return nil, errtypes.InternalError("gateway: error finding an auth provider for type: " + authType) } +/* func (s *svc) expandScopes(ctx context.Context, scopeMap map[string]*authpb.Scope) (map[string]*authpb.Scope, error) { log := appctx.GetLogger(ctx) newMap := make(map[string]*authpb.Scope) @@ -311,3 +313,4 @@ func (s *svc) statAndAddResource(ctx context.Context, r *storageprovider.Resourc return scope.AddResourceInfoScope(statResponse.Info, role, scopeMap) } +*/ diff --git a/internal/grpc/services/gateway/usershareprovider.go b/internal/grpc/services/gateway/usershareprovider.go index 7511e0a581..8a5e1c284d 100644 --- a/internal/grpc/services/gateway/usershareprovider.go +++ b/internal/grpc/services/gateway/usershareprovider.go @@ -23,11 +23,13 @@ import ( "fmt" "path" + userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" collaboration "github.com/cs3org/go-cs3apis/cs3/sharing/collaboration/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" typesv1beta1 "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" "github.com/cs3org/reva/pkg/appctx" + ctxpkg "github.com/cs3org/reva/pkg/ctx" "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/rgrpc/status" "github.com/cs3org/reva/pkg/rgrpc/todo/pool" @@ -321,7 +323,7 @@ func (s *svc) UpdateReceivedShare(ctx context.Context, req *collaboration.Update } // if we don't need to create/delete references then we return early. - if !s.c.CommitShareToStorageRef { + if !s.c.CommitShareToStorageRef || ctxpkg.ContextMustGetUser(ctx).Id.Type == userpb.UserType_USER_TYPE_LIGHTWEIGHT { return res, nil } diff --git a/pkg/app/provider/wopi/wopi.go b/pkg/app/provider/wopi/wopi.go index 5f2a38add1..0a9b2fd187 100644 --- a/pkg/app/provider/wopi/wopi.go +++ b/pkg/app/provider/wopi/wopi.go @@ -36,6 +36,7 @@ import ( "github.com/beevik/etree" appprovider "github.com/cs3org/go-cs3apis/cs3/app/provider/v1beta1" appregistry "github.com/cs3org/go-cs3apis/cs3/app/registry/v1beta1" + userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/cs3org/reva/pkg/app" "github.com/cs3org/reva/pkg/app/provider/registry" @@ -139,6 +140,11 @@ func (p *wopiProvider) GetAppURL(ctx context.Context, resource *provider.Resourc u, ok := ctxpkg.ContextGetUser(ctx) if ok { // else defaults to "Guest xyz" + if u.Id.Type == userpb.UserType_USER_TYPE_LIGHTWEIGHT { + q.Add("userid", resource.Owner.OpaqueId+"@"+resource.Owner.Idp) + } else { + q.Add("userid", u.Id.OpaqueId+"@"+u.Id.Idp) + } var isPublicShare bool if u.Opaque != nil { if _, ok := u.Opaque.Map["public-share-role"]; ok { @@ -148,7 +154,6 @@ func (p *wopiProvider) GetAppURL(ctx context.Context, resource *provider.Resourc if !isPublicShare { q.Add("username", u.Username) - q.Add("userid", u.Id.OpaqueId+"@"+u.Id.Idp) } } diff --git a/pkg/auth/manager/oidc/oidc.go b/pkg/auth/manager/oidc/oidc.go index 2ed67a2698..e882b399ba 100644 --- a/pkg/auth/manager/oidc/oidc.go +++ b/pkg/auth/manager/oidc/oidc.go @@ -23,6 +23,7 @@ package oidc import ( "context" "fmt" + "strings" "time" oidc "github.com/coreos/go-oidc" @@ -130,6 +131,12 @@ func (am *mgr) Authenticate(ctx context.Context, clientID, clientSecret string) if claims["email_verified"] == nil { // This is not set in simplesamlphp claims["email_verified"] = false } + if claims["preferred_username"] == nil { + claims["preferred_username"] = claims[am.c.IDClaim] + } + if claims["name"] == nil { + claims["name"] = claims[am.c.IDClaim] + } if claims["email"] == nil { return nil, nil, fmt.Errorf("no \"email\" attribute found in userinfo: maybe the client did not request the oidc \"email\"-scope") @@ -158,7 +165,7 @@ func (am *mgr) Authenticate(ctx context.Context, clientID, clientSecret string) userID := &user.UserId{ OpaqueId: claims[am.c.IDClaim].(string), // a stable non reassignable id Idp: claims["issuer"].(string), // in the scope of this issuer - Type: user.UserType_USER_TYPE_PRIMARY, + Type: getUserType(claims[am.c.IDClaim].(string)), } gwc, err := pool.GetGatewayServiceClient(am.c.GatewaySvc) if err != nil { @@ -236,3 +243,17 @@ func (am *mgr) getOIDCProvider(ctx context.Context) (*oidc.Provider, error) { am.provider = provider return am.provider, nil } + +func getUserType(upn string) user.UserType { + var t user.UserType + switch { + case strings.HasPrefix(upn, "guest"): + t = user.UserType_USER_TYPE_LIGHTWEIGHT + case strings.Contains(upn, "@"): + t = user.UserType_USER_TYPE_FEDERATED + default: + t = user.UserType_USER_TYPE_PRIMARY + } + return t + +} diff --git a/pkg/auth/scope/lightweight.go b/pkg/auth/scope/lightweight.go index 8256f3f09b..5ebc8d2225 100644 --- a/pkg/auth/scope/lightweight.go +++ b/pkg/auth/scope/lightweight.go @@ -20,6 +20,7 @@ package scope import ( "context" + "strings" authpb "github.com/cs3org/go-cs3apis/cs3/auth/provider/v1beta1" collaboration "github.com/cs3org/go-cs3apis/cs3/sharing/collaboration/v1beta1" @@ -34,12 +35,41 @@ func lightweightAccountScope(_ context.Context, scope *authpb.Scope, resource in // These cannot be resolved from here, but need to be added to the scope from // where the call to mint tokens is made. // From here, we only allow ListReceivedShares calls - if _, ok := resource.(*collaboration.ListReceivedSharesRequest); ok { + switch v := resource.(type) { + case *collaboration.ListReceivedSharesRequest: return true, nil + case string: + return checkLightweightPath(v), nil } return false, nil } +func checkLightweightPath(path string) bool { + paths := []string{ + "/ocs/v2.php/apps/files_sharing/api/v1/shares", + "/ocs/v1.php/apps/files_sharing/api/v1/shares", + "/ocs/v2.php/apps/files_sharing//api/v1/shares", + "/ocs/v1.php/apps/files_sharing//api/v1/shares", + "/ocs/v2.php/cloud/capabilities", + "/ocs/v1.php/cloud/capabilities", + "/ocs/v2.php/cloud/user", + "/ocs/v1.php/cloud/user", + "/remote.php/webdav", + "/remote.php/dav/files", + "/app/open", + "/app/new", + "/archiver", + "/dataprovider", + "/data", + } + for _, p := range paths { + if strings.HasPrefix(path, p) { + return true + } + } + return false +} + // AddLightweightAccountScope adds the scope to allow access to lightweight user. func AddLightweightAccountScope(role authpb.Role, scopes map[string]*authpb.Scope) (map[string]*authpb.Scope, error) { ref := &provider.Reference{Path: "/"} diff --git a/pkg/auth/scope/receivedshare.go b/pkg/auth/scope/receivedshare.go index 2974b0ca2a..81236a3c35 100644 --- a/pkg/auth/scope/receivedshare.go +++ b/pkg/auth/scope/receivedshare.go @@ -54,7 +54,10 @@ func receivedShareScope(_ context.Context, scope *authpb.Scope, resource interfa // AddReceivedShareScope adds the scope to allow access to a received user/group share and // the shared resource. func AddReceivedShareScope(share *collaboration.ReceivedShare, role authpb.Role, scopes map[string]*authpb.Scope) (map[string]*authpb.Scope, error) { - val, err := utils.MarshalProtoV1ToJSON(share) + // Create a new "scope share" to only expose the required fields to the scope. + scopeShare := &collaboration.Share{Id: share.Share.Id, Owner: share.Share.Owner, Creator: share.Share.Creator, ResourceId: share.Share.ResourceId} + + val, err := utils.MarshalProtoV1ToJSON(&collaboration.ReceivedShare{Share: scopeShare}) if err != nil { return nil, err } diff --git a/pkg/auth/scope/share.go b/pkg/auth/scope/share.go index 48683dea17..6284efd007 100644 --- a/pkg/auth/scope/share.go +++ b/pkg/auth/scope/share.go @@ -121,7 +121,10 @@ func checkSharePath(path string) bool { // AddShareScope adds the scope to allow access to a user/group share and // the shared resource. func AddShareScope(share *collaboration.Share, role authpb.Role, scopes map[string]*authpb.Scope) (map[string]*authpb.Scope, error) { - val, err := utils.MarshalProtoV1ToJSON(share) + // Create a new "scope share" to only expose the required fields to the scope. + scopeShare := &collaboration.Share{Id: share.Id, Owner: share.Owner, Creator: share.Creator, ResourceId: share.ResourceId} + + val, err := utils.MarshalProtoV1ToJSON(scopeShare) if err != nil { return nil, err } diff --git a/pkg/cbox/user/rest/rest.go b/pkg/cbox/user/rest/rest.go index eeedaba59a..fbd1b125b2 100644 --- a/pkg/cbox/user/rest/rest.go +++ b/pkg/cbox/user/rest/rest.go @@ -128,9 +128,7 @@ func (m *manager) Configure(ml map[string]interface{}) error { return nil } -func (m *manager) getUserByParam(ctx context.Context, param, val string) (map[string]interface{}, error) { - url := fmt.Sprintf("%s/Identity?filter=%s:%s&field=upn&field=primaryAccountEmail&field=displayName&field=uid&field=gid&field=type", - m.conf.APIBaseURL, param, url.QueryEscape(val)) +func (m *manager) getUser(ctx context.Context, url string) (map[string]interface{}, error) { responseData, err := m.apiTokenManager.SendAPIGetRequest(ctx, url, false) if err != nil { return nil, err @@ -151,17 +149,38 @@ func (m *manager) getUserByParam(ctx context.Context, param, val string) (map[st } if len(users) != 1 { - return nil, errors.New("rest: user not found: " + param + ": " + val) + return nil, errors.New("rest: user not found for URL: " + url) } return users[0], nil } +func (m *manager) getUserByParam(ctx context.Context, param, val string) (map[string]interface{}, error) { + url := fmt.Sprintf("%s/Identity?filter=%s:%s&field=upn&field=primaryAccountEmail&field=displayName&field=uid&field=gid&field=type", + m.conf.APIBaseURL, param, url.QueryEscape(val)) + return m.getUser(ctx, url) +} + +func (m *manager) getLightweightUser(ctx context.Context, mail string) (map[string]interface{}, error) { + url := fmt.Sprintf("%s/Identity?filter=primaryAccountEmail:%s&filter=upn:contains:guest&field=upn&field=primaryAccountEmail&field=displayName&field=uid&field=gid&field=type", + m.conf.APIBaseURL, url.QueryEscape(mail)) + return m.getUser(ctx, url) +} + func (m *manager) getInternalUserID(ctx context.Context, uid *userpb.UserId) (string, error) { internalID, err := m.fetchCachedInternalID(uid) if err != nil { - userData, err := m.getUserByParam(ctx, "upn", uid.OpaqueId) + var ( + userData map[string]interface{} + err error + ) + if uid.Type == userpb.UserType_USER_TYPE_LIGHTWEIGHT { + // Lightweight accounts need to be fetched by email + userData, err = m.getLightweightUser(ctx, strings.TrimPrefix(uid.OpaqueId, "guest:")) + } else { + userData, err = m.getUserByParam(ctx, "upn", uid.OpaqueId) + } if err != nil { return "", err } @@ -217,7 +236,16 @@ func (m *manager) parseAndCacheUser(ctx context.Context, userData map[string]int func (m *manager) GetUser(ctx context.Context, uid *userpb.UserId) (*userpb.User, error) { u, err := m.fetchCachedUserDetails(uid) if err != nil { - userData, err := m.getUserByParam(ctx, "upn", uid.OpaqueId) + var ( + userData map[string]interface{} + err error + ) + if uid.Type == userpb.UserType_USER_TYPE_LIGHTWEIGHT { + // Lightweight accounts need to be fetched by email + userData, err = m.getLightweightUser(ctx, strings.TrimPrefix(uid.OpaqueId, "guest:")) + } else { + userData, err = m.getUserByParam(ctx, "upn", uid.OpaqueId) + } if err != nil { return nil, err } @@ -252,7 +280,12 @@ func (m *manager) GetUserByClaim(ctx context.Context, claim, value string) (*use userData, err := m.getUserByParam(ctx, claim, value) if err != nil { - return nil, err + // Lightweight accounts need to be fetched by email + if strings.HasPrefix(value, "guest:") { + if userData, err = m.getLightweightUser(ctx, strings.TrimPrefix(value, "guest:")); err != nil { + return nil, err + } + } } u := m.parseAndCacheUser(ctx, userData) diff --git a/pkg/eosclient/eosbinary/eosbinary.go b/pkg/eosclient/eosbinary/eosbinary.go index 91b45dd037..0da98de2d6 100644 --- a/pkg/eosclient/eosbinary/eosbinary.go +++ b/pkg/eosclient/eosbinary/eosbinary.go @@ -1001,7 +1001,7 @@ func (c *Client) parseFileInfo(raw string) (*eosclient.FileInfo, error) { }) var previousXAttr = "" for _, p := range partsBySpace { - partsByEqual := strings.Split(p, "=") // we have kv pairs like [size 14] + partsByEqual := strings.SplitN(p, "=", 2) // we have kv pairs like [size 14] if len(partsByEqual) == 2 { // handle xattrn and xattrv special cases switch { diff --git a/pkg/storage/utils/acl/acl.go b/pkg/storage/utils/acl/acl.go index 1c493c7224..311ee1a454 100644 --- a/pkg/storage/utils/acl/acl.go +++ b/pkg/storage/utils/acl/acl.go @@ -57,7 +57,13 @@ func Parse(acls string, delimiter string) (*ACLs, error) { if t == "" || isComment(t) { continue } - entry, err := ParseEntry(t) + var err error + var entry *Entry + if strings.HasPrefix(t, TypeLightweight) { + entry, err = ParseLWEntry(t) + } else { + entry, err = ParseEntry(t) + } if err != nil { return nil, err } @@ -133,6 +139,24 @@ func ParseEntry(singleSysACL string) (*Entry, error) { }, nil } +// ParseLWEntry parses a single lightweight ACL +func ParseLWEntry(singleSysACL string) (*Entry, error) { + if !strings.HasPrefix(singleSysACL, TypeLightweight+":") { + return nil, errInvalidACL + } + singleSysACL = strings.TrimPrefix(singleSysACL, TypeLightweight+":") + + tokens := strings.Split(singleSysACL, "=") + if len(tokens) != 2 { + return nil, errInvalidACL + } + return &Entry{ + Type: TypeLightweight, + Qualifier: tokens[0], + Permissions: tokens[1], + }, nil +} + // CitrineSerialize serializes an ACL entry for citrine EOS ACLs func (a *Entry) CitrineSerialize() string { return fmt.Sprintf("%s:%s=%s", a.Type, a.Qualifier, a.Permissions) diff --git a/pkg/storage/utils/eosfs/eosfs.go b/pkg/storage/utils/eosfs/eosfs.go index 028e7d90d7..50cdc8dfd9 100644 --- a/pkg/storage/utils/eosfs/eosfs.go +++ b/pkg/storage/utils/eosfs/eosfs.go @@ -811,7 +811,18 @@ func (fs *eosfs) GetMD(ctx context.Context, ref *provider.Reference, mdKeys []st if err != nil { return nil, err } - auth, err := fs.getUserAuth(ctx, u, "") + + fn := "" + if u.Id.Type == userpb.UserType_USER_TYPE_LIGHTWEIGHT { + p, err := fs.resolve(ctx, ref) + if err != nil { + return nil, errors.Wrap(err, "eosfs: error resolving reference") + } + + fn = fs.wrap(ctx, p) + } + + auth, err := fs.getUserAuth(ctx, u, fn) if err != nil { return nil, err } @@ -838,7 +849,7 @@ func (fs *eosfs) GetMD(ctx context.Context, ref *provider.Reference, mdKeys []st } } - fn := fs.wrap(ctx, p) + fn = fs.wrap(ctx, p) eosFileInfo, err := fs.c.GetFileInfoByPath(ctx, auth, fn) if err != nil { return nil, err From 65014d1f189be7f821b8e5e5efe5b2d244bf444b Mon Sep 17 00:00:00 2001 From: Swikriti Tripathi <41103328+SwikritiT@users.noreply.github.com> Date: Fri, 14 Jan 2022 10:59:45 +0545 Subject: [PATCH 43/58] [tests-only]Bump the commit id for tests (#2441) * Bump the commit id for tests * Adding failing tests to expected to failure --- .drone.env | 2 +- tests/acceptance/expected-failures-on-OCIS-storage.md | 2 ++ tests/acceptance/expected-failures-on-S3NG-storage.md | 2 ++ 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.drone.env b/.drone.env index 86f199a58b..22bb1ee8a6 100644 --- a/.drone.env +++ b/.drone.env @@ -1,3 +1,3 @@ # The test runner source for API tests -CORE_COMMITID=09d584745d6cbd6aebc557d9b78f6130e9b99e2b +CORE_COMMITID=88dc1fc01b8afd254d59cbface6ef48670848e22 CORE_BRANCH=master diff --git a/tests/acceptance/expected-failures-on-OCIS-storage.md b/tests/acceptance/expected-failures-on-OCIS-storage.md index d3eed0341e..cf97c7a7b2 100644 --- a/tests/acceptance/expected-failures-on-OCIS-storage.md +++ b/tests/acceptance/expected-failures-on-OCIS-storage.md @@ -1350,6 +1350,8 @@ _ocs: api compatibility, return correct status code_ - [apiVersions/fileVersionAuthor.feature:101](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiVersions/fileVersionAuthor.feature#L101) - [apiVersions/fileVersionAuthor.feature:128](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiVersions/fileVersionAuthor.feature#L128) - [apiVersions/fileVersionAuthor.feature:155](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiVersions/fileVersionAuthor.feature#L155) +- [apiVersions/fileVersionAuthor.feature:180](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiVersions/fileVersionAuthor.feature#L180) +- [apiVersions/fileVersionAuthor.feature:220](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiVersions/fileVersionAuthor.feature#L220) ### [Allow public link sharing only for certain groups feature not implemented] - [apiSharePublicLink2/allowGroupToCreatePublicLinks.feature:35](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiSharePublicLink2/allowGroupToCreatePublicLinks.feature#L35) diff --git a/tests/acceptance/expected-failures-on-S3NG-storage.md b/tests/acceptance/expected-failures-on-S3NG-storage.md index 6d8ee7cf79..4fc1c4446a 100644 --- a/tests/acceptance/expected-failures-on-S3NG-storage.md +++ b/tests/acceptance/expected-failures-on-S3NG-storage.md @@ -24,6 +24,8 @@ Basic file management like up and download, move, copy, properties, quota, trash - [apiVersions/fileVersionAuthor.feature:101](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiVersions/fileVersionAuthor.feature#L101) - [apiVersions/fileVersionAuthor.feature:128](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiVersions/fileVersionAuthor.feature#L128) - [apiVersions/fileVersionAuthor.feature:155](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiVersions/fileVersionAuthor.feature#L155) +- [apiVersions/fileVersionAuthor.feature:180](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiVersions/fileVersionAuthor.feature#L180) +- [apiVersions/fileVersionAuthor.feature:220](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiVersions/fileVersionAuthor.feature#L220) #### [Getting information about a folder overwritten by a file gives 500 error instead of 404](https://github.com/owncloud/ocis/issues/1239) - [apiWebdavProperties1/copyFile.feature:226](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavProperties1/copyFile.feature#L226) From 16a4df2051cbe2367e72636f1d2ea35fb938d46b Mon Sep 17 00:00:00 2001 From: Mouratidis Theofilos Date: Mon, 17 Jan 2022 12:19:11 +0100 Subject: [PATCH 44/58] CephFS Reva v0.2 (#1209) --- .dockerignore | 1 - .drone.star | 40 ++ Dockerfile.revad-ceph | 52 ++ Makefile | 12 +- changelog/unreleased/cephfs-driver.md | 3 + .../packages/storage/fs/cephfs/_index.md | 26 + examples/ceph/ceph.conf | 3 + examples/ceph/keyring | 2 + go.mod | 4 + go.sum | 12 + pkg/storage/fs/cephfs/cephfs.go | 591 ++++++++++++++++++ pkg/storage/fs/cephfs/chunking.go | 344 ++++++++++ pkg/storage/fs/cephfs/connections.go | 315 ++++++++++ pkg/storage/fs/cephfs/errors.go | 64 ++ pkg/storage/fs/cephfs/options.go | 90 +++ pkg/storage/fs/cephfs/permissions.go | 325 ++++++++++ pkg/storage/fs/cephfs/unsupported.go | 39 ++ pkg/storage/fs/cephfs/upload.go | 413 ++++++++++++ pkg/storage/fs/cephfs/user.go | 242 +++++++ pkg/storage/fs/cephfs/utils.go | 245 ++++++++ pkg/storage/fs/loader/loader.go | 1 + 21 files changed, 2819 insertions(+), 5 deletions(-) create mode 100644 Dockerfile.revad-ceph create mode 100644 changelog/unreleased/cephfs-driver.md create mode 100644 docs/content/en/docs/config/packages/storage/fs/cephfs/_index.md create mode 100644 examples/ceph/ceph.conf create mode 100644 examples/ceph/keyring create mode 100644 pkg/storage/fs/cephfs/cephfs.go create mode 100644 pkg/storage/fs/cephfs/chunking.go create mode 100644 pkg/storage/fs/cephfs/connections.go create mode 100644 pkg/storage/fs/cephfs/errors.go create mode 100644 pkg/storage/fs/cephfs/options.go create mode 100644 pkg/storage/fs/cephfs/permissions.go create mode 100644 pkg/storage/fs/cephfs/unsupported.go create mode 100644 pkg/storage/fs/cephfs/upload.go create mode 100644 pkg/storage/fs/cephfs/user.go create mode 100644 pkg/storage/fs/cephfs/utils.go diff --git a/.dockerignore b/.dockerignore index 429a7e9d46..ab22e55d14 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,7 +1,6 @@ .github changelog docs -examples grpc-tests tests tools diff --git a/.drone.star b/.drone.star index 3c60499d7d..92460ad3e3 100644 --- a/.drone.star +++ b/.drone.star @@ -229,6 +229,26 @@ def buildAndPublishDocker(): ], }, }, + { + "name": "publish-docker-revad-ceph-latest", + "pull": "always", + "image": "plugins/docker", + "settings": { + "repo": "cs3org/revad", + "tags": "latest-ceph", + "dockerfile": "Dockerfile.revad-ceph", + "username": { + "from_secret": "dockerhub_username", + }, + "password": { + "from_secret": "dockerhub_password", + }, + "custom_dns": [ + "128.142.17.5", + "128.142.16.5", + ], + }, + }, ], } @@ -481,6 +501,26 @@ def release(): ], }, }, + { + "name": "docker-revad-ceph-tag", + "pull": "always", + "image": "plugins/docker", + "settings": { + "repo": "cs3org/revad", + "tags": "${DRONE_TAG}-ceph", + "dockerfile": "Dockerfile.revad-ceph", + "username": { + "from_secret": "dockerhub_username", + }, + "password": { + "from_secret": "dockerhub_password", + }, + "custom_dns": [ + "128.142.17.5", + "128.142.16.5", + ], + }, + }, ], "depends_on": ["changelog"], } diff --git a/Dockerfile.revad-ceph b/Dockerfile.revad-ceph new file mode 100644 index 0000000000..186acfcc63 --- /dev/null +++ b/Dockerfile.revad-ceph @@ -0,0 +1,52 @@ +# Copyright 2018-2021 CERN +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# In applying this license, CERN does not waive the privileges and immunities +# granted to it by virtue of its status as an Intergovernmental Organization +# or submit itself to any jurisdiction. + +FROM ceph/daemon-base + +RUN dnf update -y && dnf install -y \ + git \ + gcc \ + make \ + libcephfs-devel \ + librbd-devel \ + librados-devel + +ADD https://golang.org/dl/go1.16.4.linux-amd64.tar.gz \ + go1.16.4.linux-amd64.tar.gz + +RUN rm -rf /usr/local/go && \ + tar -C /usr/local -xzf go1.16.4.linux-amd64.tar.gz && \ + rm go1.16.4.linux-amd64.tar.gz + +ENV PATH /go/bin:/usr/local/go/bin:$PATH +ENV GOPATH /go + +WORKDIR /go/src/github/cs3org/reva +COPY . . +RUN mkdir -p /go/bin \ + make build-revad-cephfs-docker && \ + cp /go/src/github/cs3org/reva/cmd/revad/revad /usr/bin/revad + +RUN cp -r examples/ceph /etc/ + +RUN mkdir -p /etc/revad/ && echo "" > /etc/revad/revad.toml + +EXPOSE 9999 10000 + +ENTRYPOINT [ "/usr/bin/revad" ] +CMD [ "-c", "/etc/revad/revad.toml", "-p", "/var/run/revad.pid" ] diff --git a/Makefile b/Makefile index 465a71ef9b..b686fca1d9 100644 --- a/Makefile +++ b/Makefile @@ -30,10 +30,9 @@ off: imports: off `go env GOPATH`/bin/goimports -w tools pkg internal cmd -build: imports test-go-version - gofmt -s -w . - go build -ldflags ${BUILD_FLAGS} -o ./cmd/revad/revad ./cmd/revad - go build -ldflags ${BUILD_FLAGS} -o ./cmd/reva/reva ./cmd/reva +build: build-revad build-reva test-go-version + +build-cephfs: build-revad-cephfs build-reva tidy: go mod tidy @@ -41,6 +40,9 @@ tidy: build-revad: imports go build -ldflags ${BUILD_FLAGS} -o ./cmd/revad/revad ./cmd/revad +build-revad-cephfs: imports + go build -ldflags ${BUILD_FLAGS} -tags ceph -o ./cmd/revad/revad ./cmd/revad + build-reva: imports go build -ldflags ${BUILD_FLAGS} -o ./cmd/reva/reva ./cmd/reva @@ -104,6 +106,8 @@ ci: build-ci test lint-ci # to be run in Docker build build-revad-docker: off go build -ldflags ${BUILD_FLAGS} -o ./cmd/revad/revad ./cmd/revad +build-revad-cephfs-docker: off + go build -ldflags ${BUILD_FLAGS} -tags ceph -o ./cmd/revad/revad ./cmd/revad build-reva-docker: off go build -ldflags ${BUILD_FLAGS} -o ./cmd/reva/reva ./cmd/reva clean: diff --git a/changelog/unreleased/cephfs-driver.md b/changelog/unreleased/cephfs-driver.md new file mode 100644 index 0000000000..9015c61538 --- /dev/null +++ b/changelog/unreleased/cephfs-driver.md @@ -0,0 +1,3 @@ +Enhancement: Reva CephFS module v0.2.1 + +https://github.com/cs3org/reva/pull/1209 \ No newline at end of file diff --git a/docs/content/en/docs/config/packages/storage/fs/cephfs/_index.md b/docs/content/en/docs/config/packages/storage/fs/cephfs/_index.md new file mode 100644 index 0000000000..48dc231ebf --- /dev/null +++ b/docs/content/en/docs/config/packages/storage/fs/cephfs/_index.md @@ -0,0 +1,26 @@ +--- +title: "cephfs" +linkTitle: "cephfs" +weight: 10 +description: > + Configuration for the cephfs service +--- + +# _struct: config_ + +{{% dir name="root" type="string" default="/var/tmp/reva/" %}} +Path of root directory for user storage. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/storage/fs/cephfs/cephfs.go#L34) +{{< highlight toml >}} +[storage.fs.cephfs] +root = "/var/tmp/reva/" +{{< /highlight >}} +{{% /dir %}} + +{{% dir name="share_folder" type="string" default="/MyShares" %}} +Path for storing share references. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/storage/fs/cephfs/cephfs.go#L35) +{{< highlight toml >}} +[storage.fs.cephfs] +share_folder = "/MyShares" +{{< /highlight >}} +{{% /dir %}} + diff --git a/examples/ceph/ceph.conf b/examples/ceph/ceph.conf new file mode 100644 index 0000000000..f2daaa8de5 --- /dev/null +++ b/examples/ceph/ceph.conf @@ -0,0 +1,3 @@ +[global] + fsid = '8aaa35c4-75dc-42e5-a812-cbc1cdfd3323' + mon_host = '[v2:188.184.96.178:3300/0,v1:188.184.96.178:6789/0] [v2:188.185.88.76:3300/0,v1:188.185.88.76:6789/0] [v2:188.185.126.6:3300/0,v1:188.185.126.6:6789/0]' diff --git a/examples/ceph/keyring b/examples/ceph/keyring new file mode 100644 index 0000000000..9e555cc1b2 --- /dev/null +++ b/examples/ceph/keyring @@ -0,0 +1,2 @@ +[client.admin] + key = 'AQAu88Fg5iekGhAAeVP0Td05PuybytuRJgBRqA==' diff --git a/go.mod b/go.mod index abb2defa94..367e062000 100644 --- a/go.mod +++ b/go.mod @@ -13,11 +13,13 @@ require ( github.com/beevik/etree v1.1.0 github.com/bluele/gcache v0.0.2 github.com/c-bata/go-prompt v0.2.5 + github.com/ceph/go-ceph v0.12.0 github.com/cheggaaa/pb v1.0.29 github.com/coreos/go-oidc v2.2.1+incompatible github.com/cs3org/cato v0.0.0-20200828125504-e418fc54dd5e github.com/cs3org/go-cs3apis v0.0.0-20211214102047-7ce3134d7bf8 github.com/cubewise-code/go-mime v0.0.0-20200519001935-8c5762b177d8 + github.com/dgraph-io/ristretto v0.1.0 github.com/eventials/go-tus v0.0.0-20200718001131-45c7ec8f5d59 github.com/gdexlab/go-render v1.0.1 github.com/go-chi/chi/v5 v5.0.7 @@ -40,6 +42,7 @@ require ( github.com/jedib0t/go-pretty v4.3.0+incompatible github.com/juliangruber/go-intersect v1.1.0 github.com/mattn/go-sqlite3 v2.0.3+incompatible + github.com/maxymania/go-system v0.0.0-20170110133659-647cc364bf0b github.com/mileusna/useragent v1.0.2 github.com/minio/minio-go/v7 v7.0.20 github.com/mitchellh/copystructure v1.2.0 // indirect @@ -71,6 +74,7 @@ require ( go.opentelemetry.io/otel/trace v1.3.0 golang.org/x/crypto v0.0.0-20211215165025-cf75a172585e golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1 + golang.org/x/sync v0.0.0-20210220032951-036812b2e83c golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359 golang.org/x/term v0.0.0-20210916214954-140adaaadfaf google.golang.org/genproto v0.0.0-20211021150943-2b146023228c diff --git a/go.sum b/go.sum index b2612f2e5c..0f992f1d5a 100644 --- a/go.sum +++ b/go.sum @@ -96,6 +96,7 @@ github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d h1:Byv0BzEl github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/aws/aws-sdk-go v1.20.1/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.34.28/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48= +github.com/aws/aws-sdk-go v1.35.24/go.mod h1:tlPOdRjfxPBpNIwqDj61rmsnA85v9jc0Ps9+muhnW+k= github.com/aws/aws-sdk-go v1.38.35/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/aws/aws-sdk-go v1.40.11/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q= github.com/aws/aws-sdk-go v1.41.13/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q= @@ -118,6 +119,8 @@ github.com/c-bata/go-prompt v0.2.5 h1:3zg6PecEywxNn0xiqcXHD96fkbxghD+gdB2tbsYfl+ github.com/c-bata/go-prompt v0.2.5/go.mod h1:vFnjEGDIIA/Lib7giyE4E9c50Lvl8j0S+7FVlAwDAVw= github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/ceph/go-ceph v0.12.0 h1:nlFgKQZXOFR4oMnzXsKwTr79Y6EYDwqTrpigICGy/Tw= +github.com/ceph/go-ceph v0.12.0/go.mod h1:mafFpf5Vg8Ai8Bd+FAMvKBHLmtdpTXdRP/TNq8XWegY= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= @@ -140,6 +143,10 @@ github.com/cubewise-code/go-mime v0.0.0-20200519001935-8c5762b177d8/go.mod h1:4a github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgraph-io/ristretto v0.1.0 h1:Jv3CGQHp9OjuMBSne1485aDpUkTKEcUqF+jm/LuerPI= +github.com/dgraph-io/ristretto v0.1.0/go.mod h1:fux0lOrBhrVCJd3lcTHsIJhq1T2rokOu6v9Vcb3Q9ug= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= @@ -301,6 +308,8 @@ github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGt github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0= github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/gofrs/uuid v4.0.0+incompatible h1:1SD/1F5pU8p29ybwgQSwpQk+mwdRrXCYuPhW6m+TnJw= github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= @@ -510,6 +519,8 @@ github.com/mattn/go-tty v0.0.3 h1:5OfyWorkyO7xP52Mq7tB36ajHDG5OHrmBGIS/DtakQI= github.com/mattn/go-tty v0.0.3/go.mod h1:ihxohKRERHTVzN+aSVRwACLCeqIoZAWpoICkkvrWyR0= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/maxymania/go-system v0.0.0-20170110133659-647cc364bf0b h1:Q53idHrTuQDDHyXaxZ6pUl0I9uyD6Z6uKFK3ocX6LzI= +github.com/maxymania/go-system v0.0.0-20170110133659-647cc364bf0b/go.mod h1:KirJrATYGbTyUwVR26xIkaipRqRcMRXBf8N5dacvGus= github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= github.com/mileusna/useragent v1.0.2 h1:DgVKtiPnjxlb73z9bCwgdUvU2nQNQ97uhgfO8l9uz/w= github.com/mileusna/useragent v1.0.2/go.mod h1:3d8TOmwL/5I8pJjyVDteHtgDGcefrFUX4ccGOMKNYYc= @@ -910,6 +921,7 @@ golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501145240-bc7a7d42d5c3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/pkg/storage/fs/cephfs/cephfs.go b/pkg/storage/fs/cephfs/cephfs.go new file mode 100644 index 0000000000..e71db2df75 --- /dev/null +++ b/pkg/storage/fs/cephfs/cephfs.go @@ -0,0 +1,591 @@ +// Copyright 2018-2021 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +//go:build ceph +// +build ceph + +package cephfs + +import ( + "context" + "fmt" + "io" + "net/url" + "os" + "path/filepath" + "strconv" + "strings" + + cephfs2 "github.com/ceph/go-ceph/cephfs" + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/cs3org/reva/pkg/appctx" + "github.com/cs3org/reva/pkg/errtypes" + "github.com/cs3org/reva/pkg/storage" + "github.com/cs3org/reva/pkg/storage/fs/registry" + "github.com/mitchellh/mapstructure" + "github.com/pkg/errors" +) + +const ( + xattrTrustedNs = "trusted." + xattrEID = xattrTrustedNs + "eid" + xattrMd5 = xattrTrustedNs + "checksum" + xattrMd5ts = xattrTrustedNs + "checksumTS" + xattrRef = xattrTrustedNs + "ref" + xattrUserNs = "user." + snap = ".snap" +) + +type cephfs struct { + conf *Options + conn *connections + adminConn *adminConn + chunkHandler *ChunkHandler +} + +func init() { + registry.Register("cephfs", New) +} + +// New returns an implementation to of the storage.FS interface that talk to +// a ceph filesystem. +func New(m map[string]interface{}) (fs storage.FS, err error) { + c := &Options{} + if err = mapstructure.Decode(m, c); err != nil { + return nil, errors.Wrap(err, "error decoding conf") + } + + c.fillDefaults() + + var cache *connections + if cache, err = newCache(); err != nil { + return nil, errors.New("cephfs: can't create caches") + } + + adminConn := newAdminConn(c.IndexPool) + if adminConn == nil { + return nil, errors.Wrap(err, "cephfs: Couldn't create admin connections") + } + + for _, dir := range []string{c.ShadowFolder, c.UploadFolder} { + err = adminConn.adminMount.MakeDir(dir, dirPermFull) + if err != nil && err.Error() != errFileExists { + return nil, errors.New("cephfs: can't initialise system dir " + dir + ":" + err.Error()) + } + } + + return &cephfs{ + conf: c, + conn: cache, + adminConn: adminConn, + }, nil +} + +func (fs *cephfs) GetHome(ctx context.Context) (string, error) { + if fs.conf.DisableHome { + return "", errtypes.NotSupported("cephfs: GetHome() home supported disabled") + } + + user := fs.makeUser(ctx) + + return user.home, nil +} + +func (fs *cephfs) CreateHome(ctx context.Context) (err error) { + if fs.conf.DisableHome { + return errtypes.NotSupported("cephfs: GetHome() home supported disabled") + } + + user := fs.makeUser(ctx) + + // Stop createhome from running the whole thing because it is called multiple times + if _, err = fs.adminConn.adminMount.Statx(user.home, cephfs2.StatxMode, 0); err == nil { + return + } + + err = walkPath(user.home, func(path string) error { + return fs.adminConn.adminMount.MakeDir(path, dirPermDefault) + }, false) + if err != nil { + return getRevaError(err) + } + + err = fs.adminConn.adminMount.Chown(user.home, uint32(user.UidNumber), uint32(user.GidNumber)) + if err != nil { + return getRevaError(err) + } + + err = fs.adminConn.adminMount.SetXattr(user.home, "ceph.quota.max_bytes", []byte(fmt.Sprint(fs.conf.UserQuotaBytes)), 0) + if err != nil { + return getRevaError(err) + } + + user.op(func(cv *cacheVal) { + err = cv.mount.MakeDir(removeLeadingSlash(fs.conf.ShareFolder), dirPermDefault) + if err != nil && err.Error() == errFileExists { + err = nil + } + }) + + return getRevaError(err) +} + +func (fs *cephfs) CreateDir(ctx context.Context, ref *provider.Reference) error { + user := fs.makeUser(ctx) + path, err := user.resolveRef(ref) + if err != nil { + return getRevaError(err) + } + + user.op(func(cv *cacheVal) { + if err = cv.mount.MakeDir(path, dirPermDefault); err != nil { + return + } + + //TODO(tmourati): Add entry id logic + }) + + return getRevaError(err) +} + +func (fs *cephfs) Delete(ctx context.Context, ref *provider.Reference) (err error) { + var path string + user := fs.makeUser(ctx) + path, err = user.resolveRef(ref) + if err != nil { + return err + } + + user.op(func(cv *cacheVal) { + if err = cv.mount.Unlink(path); err != nil && err.Error() == errIsADirectory { + err = cv.mount.RemoveDir(path) + } + + //TODO(tmourati): Add entry id logic + }) + + //has already been deleted by direct mount + if err != nil && err.Error() == errNotFound { + return nil + } + + return getRevaError(err) +} + +func (fs *cephfs) Move(ctx context.Context, oldRef, newRef *provider.Reference) (err error) { + var oldPath, newPath string + user := fs.makeUser(ctx) + if oldPath, err = user.resolveRef(oldRef); err != nil { + return + } + if newPath, err = user.resolveRef(newRef); err != nil { + return + } + + user.op(func(cv *cacheVal) { + if err = cv.mount.Rename(oldPath, newPath); err != nil { + return + } + + //TODO(tmourati): Add entry id logic, handle already moved file error + }) + + // has already been moved by direct mount + if err != nil && err.Error() == errNotFound { + return nil + } + + return getRevaError(err) +} + +func (fs *cephfs) GetMD(ctx context.Context, ref *provider.Reference, mdKeys []string) (ri *provider.ResourceInfo, err error) { + var path string + user := fs.makeUser(ctx) + + if path, err = user.resolveRef(ref); err != nil { + return nil, err + } + + user.op(func(cv *cacheVal) { + var stat Statx + if stat, err = cv.mount.Statx(path, cephfs2.StatxBasicStats, 0); err != nil { + return + } + ri, err = user.fileAsResourceInfo(cv, path, stat, mdKeys) + }) + + return ri, getRevaError(err) +} + +func (fs *cephfs) ListFolder(ctx context.Context, ref *provider.Reference, mdKeys []string) (files []*provider.ResourceInfo, err error) { + var path string + user := fs.makeUser(ctx) + if path, err = user.resolveRef(ref); err != nil { + return + } + + user.op(func(cv *cacheVal) { + var dir *cephfs2.Directory + if dir, err = cv.mount.OpenDir(path); err != nil { + return + } + defer closeDir(dir) + + var entry *cephfs2.DirEntryPlus + var ri *provider.ResourceInfo + for entry, err = dir.ReadDirPlus(cephfs2.StatxBasicStats, 0); entry != nil && err == nil; entry, err = dir.ReadDirPlus(cephfs2.StatxBasicStats, 0) { + if fs.conf.HiddenDirs[entry.Name()] { + continue + } + + ri, err = user.fileAsResourceInfo(cv, filepath.Join(path, entry.Name()), entry.Statx(), mdKeys) + if ri == nil || err != nil { + if err != nil { + log := appctx.GetLogger(ctx) + log.Err(err).Msg("cephfs: error in file as resource info") + } + err = nil + continue + } + + files = append(files, ri) + } + }) + + return files, getRevaError(err) +} + +func (fs *cephfs) Download(ctx context.Context, ref *provider.Reference) (rc io.ReadCloser, err error) { + var path string + user := fs.makeUser(ctx) + if path, err = user.resolveRef(ref); err != nil { + return nil, errors.Wrap(err, "cephfs: error resolving ref") + } + + user.op(func(cv *cacheVal) { + if strings.HasPrefix(strings.TrimPrefix(path, user.home), fs.conf.ShareFolder) { + err = errtypes.PermissionDenied("cephfs: cannot download under the virtual share folder") + return + } + rc, err = cv.mount.Open(path, os.O_RDONLY, 0) + }) + + return rc, getRevaError(err) +} + +func (fs *cephfs) ListRevisions(ctx context.Context, ref *provider.Reference) (fvs []*provider.FileVersion, err error) { + //TODO(tmourati): Fix entry id logic + var path string + user := fs.makeUser(ctx) + if path, err = user.resolveRef(ref); err != nil { + return nil, errors.Wrap(err, "cephfs: error resolving ref") + } + + user.op(func(cv *cacheVal) { + if strings.HasPrefix(path, removeLeadingSlash(fs.conf.ShareFolder)) { + err = errtypes.PermissionDenied("cephfs: cannot download under the virtual share folder") + return + } + var dir *cephfs2.Directory + if dir, err = cv.mount.OpenDir(".snap"); err != nil { + return + } + defer closeDir(dir) + + for d, _ := dir.ReadDir(); d != nil; d, _ = dir.ReadDir() { + var revPath string + var stat Statx + var e error + + if strings.HasPrefix(d.Name(), ".") { + continue + } + + revPath, e = resolveRevRef(cv.mount, ref, d.Name()) + if e != nil { + continue + } + stat, e = cv.mount.Statx(revPath, cephfs2.StatxMtime|cephfs2.StatxSize, 0) + if e != nil { + continue + } + fvs = append(fvs, &provider.FileVersion{ + Key: d.Name(), + Size: stat.Size, + Mtime: uint64(stat.Mtime.Sec), + }) + } + }) + + return fvs, getRevaError(err) +} + +func (fs *cephfs) DownloadRevision(ctx context.Context, ref *provider.Reference, key string) (file io.ReadCloser, err error) { + //TODO(tmourati): Fix entry id logic + user := fs.makeUser(ctx) + + user.op(func(cv *cacheVal) { + var revPath string + revPath, err = resolveRevRef(cv.mount, ref, key) + if err != nil { + return + } + + file, err = cv.mount.Open(revPath, os.O_RDONLY, 0) + }) + + return file, getRevaError(err) +} + +func (fs *cephfs) RestoreRevision(ctx context.Context, ref *provider.Reference, key string) (err error) { + //TODO(tmourati): Fix entry id logic + var path string + user := fs.makeUser(ctx) + if path, err = user.resolveRef(ref); err != nil { + return errors.Wrap(err, "cephfs: error resolving ref") + } + + user.op(func(cv *cacheVal) { + var revPath string + if revPath, err = resolveRevRef(cv.mount, ref, key); err != nil { + err = errors.Wrap(err, "cephfs: error resolving revision ref "+ref.String()) + return + } + + var src, dst *cephfs2.File + if src, err = cv.mount.Open(revPath, os.O_RDONLY, 0); err != nil { + return + } + defer closeFile(src) + + if dst, err = cv.mount.Open(path, os.O_WRONLY|os.O_TRUNC, 0); err != nil { + return + } + defer closeFile(dst) + + _, err = io.Copy(dst, src) + }) + + return getRevaError(err) +} + +func (fs *cephfs) GetPathByID(ctx context.Context, id *provider.ResourceId) (str string, err error) { + //TODO(tmourati): Add entry id logic + return "", errtypes.NotSupported("cephfs: entry IDs currently not supported") +} + +func (fs *cephfs) AddGrant(ctx context.Context, ref *provider.Reference, g *provider.Grant) (err error) { + var path string + user := fs.makeUser(ctx) + if path, err = user.resolveRef(ref); err != nil { + return + } + + user.op(func(cv *cacheVal) { + err = fs.changePerms(ctx, cv.mount, g, path, updateGrant) + }) + + return getRevaError(err) +} + +func (fs *cephfs) RemoveGrant(ctx context.Context, ref *provider.Reference, g *provider.Grant) (err error) { + var path string + user := fs.makeUser(ctx) + if path, err = user.resolveRef(ref); err != nil { + return + } + + user.op(func(cv *cacheVal) { + err = fs.changePerms(ctx, cv.mount, g, path, removeGrant) + }) + + return getRevaError(err) +} + +func (fs *cephfs) UpdateGrant(ctx context.Context, ref *provider.Reference, g *provider.Grant) (err error) { + var path string + user := fs.makeUser(ctx) + if path, err = user.resolveRef(ref); err != nil { + return + } + + user.op(func(cv *cacheVal) { + err = fs.changePerms(ctx, cv.mount, g, path, updateGrant) + }) + + return getRevaError(err) +} + +func (fs *cephfs) DenyGrant(ctx context.Context, ref *provider.Reference, g *provider.Grantee) (err error) { + var path string + user := fs.makeUser(ctx) + if path, err = user.resolveRef(ref); err != nil { + return + } + + user.op(func(cv *cacheVal) { + grant := &provider.Grant{Grantee: g} //nil perms will remove the whole grant + err = fs.changePerms(ctx, cv.mount, grant, path, removeGrant) + }) + + return getRevaError(err) +} + +func (fs *cephfs) ListGrants(ctx context.Context, ref *provider.Reference) (glist []*provider.Grant, err error) { + var path string + user := fs.makeUser(ctx) + if path, err = user.resolveRef(ref); err != nil { + return + } + + user.op(func(cv *cacheVal) { + glist = fs.getFullPermissionSet(ctx, cv.mount, path) + + if glist == nil { + err = errors.New("cephfs: error listing grants on " + path) + } + }) + + return glist, getRevaError(err) +} + +func (fs *cephfs) GetQuota(ctx context.Context, ref *provider.Reference) (total uint64, used uint64, err error) { + user := fs.makeUser(ctx) + + log := appctx.GetLogger(ctx) + user.op(func(cv *cacheVal) { + var buf []byte + buf, err = cv.mount.GetXattr(".", "ceph.quota.max_bytes") + if err != nil { + log.Warn().Msg("cephfs: user quota bytes not set") + total = fs.conf.UserQuotaBytes + } else { + total, _ = strconv.ParseUint(string(buf), 10, 64) + } + + buf, err = cv.mount.GetXattr(".", "ceph.dir.rbytes") + if err == nil { + used, err = strconv.ParseUint(string(buf), 10, 64) + } + }) + + return total, used, getRevaError(err) +} + +func (fs *cephfs) CreateReference(ctx context.Context, path string, targetURI *url.URL) (err error) { + user := fs.makeUser(ctx) + + user.op(func(cv *cacheVal) { + if !strings.HasPrefix(strings.TrimPrefix(path, user.home), fs.conf.ShareFolder) { + err = errors.New("cephfs: can't create reference outside a share folder") + } else { + err = cv.mount.MakeDir(path, dirPermDefault) + } + }) + if err != nil { + return getRevaError(err) + } + + user.op(func(cv *cacheVal) { + err = cv.mount.SetXattr(path, xattrRef, []byte(targetURI.String()), 0) + }) + + return getRevaError(err) +} + +func (fs *cephfs) Shutdown(ctx context.Context) (err error) { + ctx.Done() + fs.conn.clearCache() + _ = fs.adminConn.adminMount.Unmount() + _ = fs.adminConn.adminMount.Release() + fs.adminConn.radosConn.Shutdown() + + return +} + +func (fs *cephfs) SetArbitraryMetadata(ctx context.Context, ref *provider.Reference, md *provider.ArbitraryMetadata) (err error) { + var path string + user := fs.makeUser(ctx) + if path, err = user.resolveRef(ref); err != nil { + return err + } + + user.op(func(cv *cacheVal) { + for k, v := range md.Metadata { + if !strings.HasPrefix(k, xattrUserNs) { + k = xattrUserNs + k + } + if e := cv.mount.SetXattr(path, k, []byte(v), 0); e != nil { + err = errors.Wrap(err, e.Error()) + return + } + } + }) + + return getRevaError(err) +} + +func (fs *cephfs) UnsetArbitraryMetadata(ctx context.Context, ref *provider.Reference, keys []string) (err error) { + var path string + user := fs.makeUser(ctx) + if path, err = user.resolveRef(ref); err != nil { + return err + } + + user.op(func(cv *cacheVal) { + for _, key := range keys { + if !strings.HasPrefix(key, xattrUserNs) { + key = xattrUserNs + key + } + if e := cv.mount.RemoveXattr(path, key); e != nil { + err = errors.Wrap(err, e.Error()) + return + } + } + }) + + return getRevaError(err) +} + +func (fs *cephfs) EmptyRecycle(ctx context.Context) error { + return errtypes.NotSupported("cephfs: empty recycle not supported") +} + +func (fs *cephfs) CreateStorageSpace(ctx context.Context, req *provider.CreateStorageSpaceRequest) (r *provider.CreateStorageSpaceResponse, err error) { + return nil, errors.New("cephfs: createStorageSpace not supported") +} + +func (fs *cephfs) ListRecycle(ctx context.Context, basePath, key, relativePath string) ([]*provider.RecycleItem, error) { + panic("implement me") +} + +func (fs *cephfs) RestoreRecycleItem(ctx context.Context, basePath, key, relativePath string, restoreRef *provider.Reference) error { + return errors.New("cephfs: restoreRecycleItem not supported") +} + +func (fs *cephfs) PurgeRecycleItem(ctx context.Context, basePath, key, relativePath string) error { + return errors.New("cephfs: purgeRecycleItem not supported") +} + +func (fs *cephfs) ListStorageSpaces(ctx context.Context, filter []*provider.ListStorageSpacesRequest_Filter, permissions map[string]struct{}) ([]*provider.StorageSpace, error) { + return nil, errors.New("cephfs: listStorageSpaces not supported") +} + +func (fs *cephfs) UpdateStorageSpace(ctx context.Context, req *provider.UpdateStorageSpaceRequest) (*provider.UpdateStorageSpaceResponse, error) { + return nil, errors.New("cephfs: updateStorageSpace not supported") +} diff --git a/pkg/storage/fs/cephfs/chunking.go b/pkg/storage/fs/cephfs/chunking.go new file mode 100644 index 0000000000..bb4e48fe2c --- /dev/null +++ b/pkg/storage/fs/cephfs/chunking.go @@ -0,0 +1,344 @@ +// Copyright 2018-2021 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +//go:build ceph +// +build ceph + +package cephfs + +import ( + "context" + "fmt" + "io" + "os" + "path/filepath" + "regexp" + "strconv" + "strings" + "time" + + cephfs2 "github.com/ceph/go-ceph/cephfs" + "github.com/google/uuid" +) + +// IsChunked checks if a given path refers to a chunk or not +func IsChunked(fn string) (bool, error) { + // FIXME: also need to check whether the OC-Chunked header is set + return regexp.MatchString(`-chunking-\w+-[0-9]+-[0-9]+$`, fn) +} + +// ChunkBLOBInfo stores info about a particular chunk +type ChunkBLOBInfo struct { + Path string + TransferID string + TotalChunks int + CurrentChunk int +} + +// Not using the resource path in the chunk folder name allows uploading to +// the same folder after a move without having to restart the chunk upload +func (c *ChunkBLOBInfo) uploadID() string { + return fmt.Sprintf("chunking-%s-%d", c.TransferID, c.TotalChunks) +} + +// GetChunkBLOBInfo decodes a chunk name to retrieve info about it. +func GetChunkBLOBInfo(path string) (*ChunkBLOBInfo, error) { + parts := strings.Split(path, "-chunking-") + tail := strings.Split(parts[1], "-") + + totalChunks, err := strconv.Atoi(tail[1]) + if err != nil { + return nil, err + } + + currentChunk, err := strconv.Atoi(tail[2]) + if err != nil { + return nil, err + } + if currentChunk >= totalChunks { + return nil, fmt.Errorf("current chunk:%d exceeds total number of chunks:%d", currentChunk, totalChunks) + } + + return &ChunkBLOBInfo{ + Path: parts[0], + TransferID: tail[0], + TotalChunks: totalChunks, + CurrentChunk: currentChunk, + }, nil +} + +// ChunkHandler manages chunked uploads, storing the chunks in a temporary directory +// until it gets the final chunk which is then returned. +type ChunkHandler struct { + user *User + chunkFolder string +} + +// NewChunkHandler creates a handler for chunked uploads. +func NewChunkHandler(ctx context.Context, fs *cephfs) *ChunkHandler { + return &ChunkHandler{fs.makeUser(ctx), fs.conf.UploadFolder} +} + +func (c *ChunkHandler) getChunkTempFileName() string { + return fmt.Sprintf("__%d_%s", time.Now().Unix(), uuid.New().String()) +} + +func (c *ChunkHandler) getChunkFolderName(i *ChunkBLOBInfo) (path string, err error) { + path = filepath.Join(c.chunkFolder, i.uploadID()) + c.user.op(func(cv *cacheVal) { + err = cv.mount.MakeDir(path, 0777) + }) + + return +} + +func (c *ChunkHandler) saveChunk(path string, r io.ReadCloser) (finish bool, chunk string, err error) { + var chunkInfo *ChunkBLOBInfo + + chunkInfo, err = GetChunkBLOBInfo(path) + if err != nil { + err = fmt.Errorf("error getting chunk info from path: %s", path) + return + } + + chunkTempFilename := c.getChunkTempFileName() + c.user.op(func(cv *cacheVal) { + var tmpFile *cephfs2.File + target := filepath.Join(c.chunkFolder, chunkTempFilename) + tmpFile, err = cv.mount.Open(target, os.O_CREATE|os.O_WRONLY, filePermDefault) + defer closeFile(tmpFile) + if err != nil { + return + } + _, err = io.Copy(tmpFile, r) + }) + if err != nil { + return + } + + chunksFolderName, err := c.getChunkFolderName(chunkInfo) + if err != nil { + return + } + // c.logger.Info().Log("chunkfolder", chunksFolderName) + + chunkTarget := filepath.Join(chunksFolderName, strconv.Itoa(chunkInfo.CurrentChunk)) + c.user.op(func(cv *cacheVal) { + err = cv.mount.Rename(chunkTempFilename, chunkTarget) + }) + if err != nil { + return + } + + // Check that all chunks are uploaded. + // This is very inefficient, the server has to check that it has all the + // chunks after each uploaded chunk. + // A two-phase upload like DropBox is better, because the server will + // assembly the chunks when the client asks for it. + numEntries := 0 + c.user.op(func(cv *cacheVal) { + var dir *cephfs2.Directory + var entry *cephfs2.DirEntry + var chunkFile, assembledFile *cephfs2.File + + dir, err = cv.mount.OpenDir(chunksFolderName) + defer closeDir(dir) + + for entry, err = dir.ReadDir(); entry != nil && err == nil; entry, err = dir.ReadDir() { + numEntries++ + } + // to remove . and .. + numEntries -= 2 + + if err != nil || numEntries < chunkInfo.TotalChunks { + return + } + + chunk = filepath.Join(c.chunkFolder, c.getChunkTempFileName()) + assembledFile, err = cv.mount.Open(chunk, os.O_CREATE|os.O_WRONLY, filePermDefault) + defer closeFile(assembledFile) + defer deleteFile(cv.mount, chunk) + if err != nil { + return + } + + for i := 0; i < numEntries; i++ { + target := filepath.Join(chunksFolderName, strconv.Itoa(i)) + + chunkFile, err = cv.mount.Open(target, os.O_RDONLY, 0) + if err != nil { + return + } + _, err = io.Copy(assembledFile, chunkFile) + closeFile(chunkFile) + if err != nil { + return + } + } + + // necessary approach in case assembly fails + for i := 0; i < numEntries; i++ { + target := filepath.Join(chunksFolderName, strconv.Itoa(i)) + err = cv.mount.Unlink(target) + if err != nil { + return + } + } + _ = cv.mount.Unlink(chunksFolderName) + }) + + return true, chunk, nil +} + +// WriteChunk saves an intermediate chunk temporarily and assembles all chunks +// once the final one is received. +func (c *ChunkHandler) WriteChunk(fn string, r io.ReadCloser) (string, string, error) { + finish, chunk, err := c.saveChunk(fn, r) + if err != nil { + return "", "", err + } + + if !finish { + return "", "", nil + } + + chunkInfo, err := GetChunkBLOBInfo(fn) + if err != nil { + return "", "", err + } + + return chunkInfo.Path, chunk, nil + + // TODO(labkode): implement old chunking + + /* + req2 := &provider.StartWriteSessionRequest{} + res2, err := client.StartWriteSession(ctx, req2) + if err != nil { + logger.Error(ctx, err) + w.WriteHeader(http.StatusInternalServerError) + return + } + + if res2.Status.Code != rpc.Code_CODE_OK { + logger.Println(ctx, res2.Status) + w.WriteHeader(http.StatusInternalServerError) + return + } + + sessID := res2.SessionId + logger.Build().Str("sessID", sessID).Msg(ctx, "got write session id") + + stream, err := client.Write(ctx) + if err != nil { + logger.Error(ctx, err) + w.WriteHeader(http.StatusInternalServerError) + return + } + + buffer := make([]byte, 1024*1024*3) + var offset uint64 + var numChunks uint64 + + for { + n, err := fd.Read(buffer) + if n > 0 { + req := &provider.WriteRequest{Data: buffer, Length: uint64(n), SessionId: sessID, Offset: offset} + err = stream.Send(req) + if err != nil { + logger.Error(ctx, err) + w.WriteHeader(http.StatusInternalServerError) + return + } + + numChunks++ + offset += uint64(n) + } + + if err == io.EOF { + break + } + + if err != nil { + logger.Error(ctx, err) + w.WriteHeader(http.StatusInternalServerError) + return + } + } + + res3, err := stream.CloseAndRecv() + if err != nil { + logger.Error(ctx, err) + w.WriteHeader(http.StatusInternalServerError) + return + } + + if res3.Status.Code != rpc.Code_CODE_OK { + logger.Println(ctx, err) + w.WriteHeader(http.StatusInternalServerError) + return + } + + req4 := &provider.FinishWriteSessionRequest{Filename: chunkInfo.path, SessionId: sessID} + res4, err := client.FinishWriteSession(ctx, req4) + if err != nil { + logger.Error(ctx, err) + w.WriteHeader(http.StatusInternalServerError) + return + } + + if res4.Status.Code != rpc.Code_CODE_OK { + logger.Println(ctx, res4.Status) + w.WriteHeader(http.StatusInternalServerError) + return + } + + req.Filename = chunkInfo.path + res, err = client.Stat(ctx, req) + if err != nil { + logger.Error(ctx, err) + w.WriteHeader(http.StatusInternalServerError) + return + } + + if res.Status.Code != rpc.Code_CODE_OK { + logger.Println(ctx, res.Status) + w.WriteHeader(http.StatusInternalServerError) + return + } + + md2 := res.Metadata + + w.Header().Add("Content-Type", md2.Mime) + w.Header().Set("ETag", md2.Etag) + w.Header().Set("OC-FileId", md2.Id) + w.Header().Set("OC-ETag", md2.Etag) + t := time.Unix(int64(md2.Mtime), 0) + lastModifiedString := t.Format(time.RFC1123Z) + w.Header().Set("Last-Modified", lastModifiedString) + w.Header().Set("X-OC-MTime", "accepted") + + if md == nil { + w.WriteHeader(http.StatusCreated) + return + } + + w.WriteHeader(http.StatusNoContent) + return + */ +} diff --git a/pkg/storage/fs/cephfs/connections.go b/pkg/storage/fs/cephfs/connections.go new file mode 100644 index 0000000000..7b928eaa63 --- /dev/null +++ b/pkg/storage/fs/cephfs/connections.go @@ -0,0 +1,315 @@ +// Copyright 2018-2021 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +//go:build ceph +// +build ceph + +package cephfs + +import ( + "context" + "fmt" + "time" + + "github.com/ceph/go-ceph/cephfs/admin" + rados2 "github.com/ceph/go-ceph/rados" + grouppb "github.com/cs3org/go-cs3apis/cs3/identity/group/v1beta1" + userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" + rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" + "github.com/cs3org/reva/pkg/rgrpc/todo/pool" + "github.com/pkg/errors" + + cephfs2 "github.com/ceph/go-ceph/cephfs" + "github.com/dgraph-io/ristretto" + "golang.org/x/sync/semaphore" +) + +type cacheVal struct { + perm *cephfs2.UserPerm + mount *cephfs2.MountInfo +} + +//TODO: Add to cephfs obj + +type connections struct { + cache *ristretto.Cache + lock *semaphore.Weighted + ctx context.Context + userCache *ristretto.Cache + groupCache *ristretto.Cache +} + +//TODO: make configurable/add to options +var usrLimit int64 = 1e4 + +func newCache() (c *connections, err error) { + cache, err := ristretto.NewCache(&ristretto.Config{ + NumCounters: 1e7, + MaxCost: usrLimit, + BufferItems: 64, + OnEvict: func(item *ristretto.Item) { + v := item.Value.(cacheVal) + v.perm.Destroy() + _ = v.mount.Unmount() + _ = v.mount.Release() + }, + }) + if err != nil { + return + } + + ucache, err := ristretto.NewCache(&ristretto.Config{ + NumCounters: 1e7, + MaxCost: 10 * usrLimit, + BufferItems: 64, + }) + if err != nil { + return + } + + gcache, err := ristretto.NewCache(&ristretto.Config{ + NumCounters: 1e7, + MaxCost: 10 * usrLimit, + BufferItems: 64, + }) + if err != nil { + return + } + + c = &connections{ + cache: cache, + lock: semaphore.NewWeighted(usrLimit), + ctx: context.Background(), + userCache: ucache, + groupCache: gcache, + } + + return +} + +func (c *connections) clearCache() { + c.cache.Clear() + c.cache.Close() +} + +type adminConn struct { + indexPoolName string + subvolAdmin *admin.FSAdmin + adminMount Mount + radosConn *rados2.Conn + radosIO *rados2.IOContext +} + +func newAdminConn(poolName string) *adminConn { + rados, err := rados2.NewConn() + if err != nil { + return nil + } + if err = rados.ReadDefaultConfigFile(); err != nil { + return nil + } + + if err = rados.Connect(); err != nil { + return nil + } + + pools, err := rados.ListPools() + if err != nil { + rados.Shutdown() + return nil + } + + var radosIO *rados2.IOContext + if in(poolName, pools) { + radosIO, err = rados.OpenIOContext(poolName) + if err != nil { + rados.Shutdown() + return nil + } + } else { + err = rados.MakePool(poolName) + if err != nil { + rados.Shutdown() + return nil + } + radosIO, err = rados.OpenIOContext(poolName) + if err != nil { + rados.Shutdown() + return nil + } + } + + mount, err := cephfs2.CreateFromRados(rados) + if err != nil { + rados.Shutdown() + return nil + } + + if err = mount.Mount(); err != nil { + rados.Shutdown() + destroyCephConn(mount, nil) + return nil + } + + return &adminConn{ + poolName, + admin.NewFromConn(rados), + mount, + rados, + radosIO, + } +} + +func newConn(user *User) *cacheVal { + var perm *cephfs2.UserPerm + mount, err := cephfs2.CreateMount() + if err != nil { + return destroyCephConn(mount, perm) + } + if err = mount.ReadDefaultConfigFile(); err != nil { + return destroyCephConn(mount, perm) + } + if err = mount.Init(); err != nil { + return destroyCephConn(mount, perm) + } + + if user != nil { //nil creates admin conn + perm = cephfs2.NewUserPerm(int(user.UidNumber), int(user.GidNumber), []int{}) + if err = mount.SetMountPerms(perm); err != nil { + return destroyCephConn(mount, perm) + } + } + + if err = mount.MountWithRoot("/"); err != nil { + return destroyCephConn(mount, perm) + } + + if user != nil { + if err = mount.ChangeDir(user.fs.conf.Root); err != nil { + return destroyCephConn(mount, perm) + } + } + + return &cacheVal{ + perm: perm, + mount: mount, + } +} + +func (fs *cephfs) getUserByID(ctx context.Context, uid string) (*userpb.User, error) { + if entity, found := fs.conn.userCache.Get(uid); found { + return entity.(*userpb.User), nil + } + + client, err := pool.GetGatewayServiceClient(fs.conf.GatewaySvc) + if err != nil { + return nil, errors.Wrap(err, "cephfs: error getting gateway grpc client") + } + getUserResp, err := client.GetUserByClaim(ctx, &userpb.GetUserByClaimRequest{ + Claim: "uid", + Value: uid, + }) + + if err != nil { + return nil, errors.Wrap(err, "cephfs: error getting user") + } + if getUserResp.Status.Code != rpc.Code_CODE_OK { + return nil, errors.Wrap(err, "cephfs: grpc get user failed") + } + fs.conn.userCache.SetWithTTL(uid, getUserResp.User, 1, 24*time.Hour) + fs.conn.userCache.SetWithTTL(getUserResp.User.Id.OpaqueId, getUserResp.User, 1, 24*time.Hour) + + return getUserResp.User, nil +} + +func (fs *cephfs) getUserByOpaqueID(ctx context.Context, oid string) (*userpb.User, error) { + if entity, found := fs.conn.userCache.Get(oid); found { + return entity.(*userpb.User), nil + } + client, err := pool.GetGatewayServiceClient(fs.conf.GatewaySvc) + if err != nil { + return nil, errors.Wrap(err, "cephfs: error getting gateway grpc client") + } + getUserResp, err := client.GetUser(ctx, &userpb.GetUserRequest{ + UserId: &userpb.UserId{ + OpaqueId: oid, + }, + }) + + if err != nil { + return nil, errors.Wrap(err, "cephfs: error getting user") + } + if getUserResp.Status.Code != rpc.Code_CODE_OK { + return nil, errors.Wrap(err, "cephfs: grpc get user failed") + } + fs.conn.userCache.SetWithTTL(fmt.Sprint(getUserResp.User.UidNumber), getUserResp.User, 1, 24*time.Hour) + fs.conn.userCache.SetWithTTL(oid, getUserResp.User, 1, 24*time.Hour) + + return getUserResp.User, nil +} + +func (fs *cephfs) getGroupByID(ctx context.Context, gid string) (*grouppb.Group, error) { + if entity, found := fs.conn.groupCache.Get(gid); found { + return entity.(*grouppb.Group), nil + } + + client, err := pool.GetGatewayServiceClient(fs.conf.GatewaySvc) + if err != nil { + return nil, errors.Wrap(err, "cephfs: error getting gateway grpc client") + } + getGroupResp, err := client.GetGroupByClaim(ctx, &grouppb.GetGroupByClaimRequest{ + Claim: "gid", + Value: gid, + }) + if err != nil { + return nil, errors.Wrap(err, "cephfs: error getting group") + } + if getGroupResp.Status.Code != rpc.Code_CODE_OK { + return nil, errors.Wrap(err, "cephfs: grpc get group failed") + } + fs.conn.groupCache.SetWithTTL(gid, getGroupResp.Group, 1, 24*time.Hour) + fs.conn.groupCache.SetWithTTL(getGroupResp.Group.Id.OpaqueId, getGroupResp.Group, 1, 24*time.Hour) + + return getGroupResp.Group, nil +} + +func (fs *cephfs) getGroupByOpaqueID(ctx context.Context, oid string) (*grouppb.Group, error) { + if entity, found := fs.conn.groupCache.Get(oid); found { + return entity.(*grouppb.Group), nil + } + client, err := pool.GetGatewayServiceClient(fs.conf.GatewaySvc) + if err != nil { + return nil, errors.Wrap(err, "cephfs: error getting gateway grpc client") + } + getGroupResp, err := client.GetGroup(ctx, &grouppb.GetGroupRequest{ + GroupId: &grouppb.GroupId{ + OpaqueId: oid, + }, + }) + + if err != nil { + return nil, errors.Wrap(err, "cephfs: error getting group") + } + if getGroupResp.Status.Code != rpc.Code_CODE_OK { + return nil, errors.Wrap(err, "cephfs: grpc get group failed") + } + fs.conn.userCache.SetWithTTL(fmt.Sprint(getGroupResp.Group.GidNumber), getGroupResp.Group, 1, 24*time.Hour) + fs.conn.userCache.SetWithTTL(oid, getGroupResp.Group, 1, 24*time.Hour) + + return getGroupResp.Group, nil +} diff --git a/pkg/storage/fs/cephfs/errors.go b/pkg/storage/fs/cephfs/errors.go new file mode 100644 index 0000000000..a4ab013c97 --- /dev/null +++ b/pkg/storage/fs/cephfs/errors.go @@ -0,0 +1,64 @@ +// Copyright 2018-2021 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +//go:build ceph +// +build ceph + +package cephfs + +/* + #include + #include + #include +*/ +import "C" +import ( + "fmt" + + "github.com/cs3org/reva/pkg/errtypes" +) + +func wrapErrorMsg(code C.int) string { + return fmt.Sprintf("cephfs: ret=-%d, %s", code, C.GoString(C.strerror(code))) +} + +var ( + errNotFound = wrapErrorMsg(C.ENOENT) + errFileExists = wrapErrorMsg(C.EEXIST) + errNoSpaceLeft = wrapErrorMsg(C.ENOSPC) + errIsADirectory = wrapErrorMsg(C.EISDIR) + errPermissionDenied = wrapErrorMsg(C.EACCES) +) + +func getRevaError(err error) error { + if err == nil { + return nil + } + switch err.Error() { + case errNotFound: + return errtypes.NotFound("cephfs: dir entry not found") + case errPermissionDenied: + return errtypes.PermissionDenied("cephfs: permission denied") + case errFileExists: + return errtypes.AlreadyExists("cephfs: file already exists") + case errNoSpaceLeft: + return errtypes.InsufficientStorage("cephfs: no space left on device") + default: + return errtypes.InternalError(err.Error()) + } +} diff --git a/pkg/storage/fs/cephfs/options.go b/pkg/storage/fs/cephfs/options.go new file mode 100644 index 0000000000..0b4b81f0e6 --- /dev/null +++ b/pkg/storage/fs/cephfs/options.go @@ -0,0 +1,90 @@ +// Copyright 2018-2021 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +//go:build ceph +// +build ceph + +package cephfs + +import ( + "path/filepath" + + "github.com/cs3org/reva/pkg/sharedconf" +) + +// Options for the cephfs module +type Options struct { + GatewaySvc string `mapstructure:"gatewaysvc"` + IndexPool string `mapstructure:"index_pool"` + Root string `mapstructure:"root"` + ShadowFolder string `mapstructure:"shadow_folder"` + ShareFolder string `mapstructure:"share_folder"` + UploadFolder string `mapstructure:"uploads"` + UserLayout string `mapstructure:"user_layout"` + + DisableHome bool `mapstructure:"disable_home"` + UserQuotaBytes uint64 `mapstructure:"user_quota_bytes"` + HiddenDirs map[string]bool +} + +func (c *Options) fillDefaults() { + c.GatewaySvc = sharedconf.GetGatewaySVC(c.GatewaySvc) + + if c.IndexPool == "" { + c.IndexPool = "path_index" + } + + if c.Root == "" { + c.Root = "/home" + } else { + c.Root = addLeadingSlash(c.Root) //force absolute path in case leading "/" is omitted + } + + if c.ShadowFolder == "" { + c.ShadowFolder = "/.reva_hidden" + } else { + c.ShadowFolder = addLeadingSlash(c.ShadowFolder) + } + + if c.ShareFolder == "" { + c.ShareFolder = "/Shares" + } else { + c.ShareFolder = addLeadingSlash(c.ShareFolder) + } + + if c.UploadFolder == "" { + c.UploadFolder = ".uploads" + } + c.UploadFolder = filepath.Join(c.ShadowFolder, c.UploadFolder) + + if c.UserLayout == "" { + c.UserLayout = "{{.Username}}" + } + + c.HiddenDirs = map[string]bool{ + ".": true, + "..": true, + removeLeadingSlash(c.ShadowFolder): true, + } + + c.DisableHome = false // it is currently only home based + + if c.UserQuotaBytes == 0 { + c.UserQuotaBytes = 50000000000 + } +} diff --git a/pkg/storage/fs/cephfs/permissions.go b/pkg/storage/fs/cephfs/permissions.go new file mode 100644 index 0000000000..8b4a32484d --- /dev/null +++ b/pkg/storage/fs/cephfs/permissions.go @@ -0,0 +1,325 @@ +// Copyright 2018-2021 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +//go:build ceph +// +build ceph + +package cephfs + +import ( + "context" + "errors" + "fmt" + "strings" + + userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" + + cephfs2 "github.com/ceph/go-ceph/cephfs" + grouppb "github.com/cs3org/go-cs3apis/cs3/identity/group/v1beta1" + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/maxymania/go-system/posix_acl" +) + +var perms = map[rune][]string{ + 'r': { + "Stat", + "GetPath", + "GetQuota", + "InitiateFileDownload", + "ListGrants", + }, + 'w': { + "AddGrant", + "CreateContainer", + "Delete", + "InitiateFileUpload", + "Move", + "RemoveGrant", + "PurgeRecycle", + "RestoreFileVersion", + "RestoreRecycleItem", + "UpdateGrant", + }, + 'x': { + "ListRecycle", + "ListContainer", + "ListFileVersions", + }, +} + +const ( + aclXattr = "system.posix_acl_access" +) + +var op2int = map[rune]uint16{'r': 4, 'w': 2, 'x': 1} + +func getPermissionSet(user *User, stat *cephfs2.CephStatx, mount Mount, path string) (perm *provider.ResourcePermissions) { + perm = &provider.ResourcePermissions{} + + if int64(stat.Uid) == user.UidNumber || int64(stat.Gid) == user.GidNumber { + updatePerms(perm, "rwx", false) + return + } + + acls := &posix_acl.Acl{} + var xattr []byte + var err error + if xattr, err = mount.GetXattr(path, aclXattr); err != nil { + return nil + } + acls.Decode(xattr) + + group, err := user.fs.getGroupByID(user.ctx, fmt.Sprint(stat.Gid)) + + for _, acl := range acls.List { + rwx := strings.Split(acl.String(), ":")[2] + switch acl.GetType() { + case posix_acl.ACL_USER: + if int64(acl.GetID()) == user.UidNumber { + updatePerms(perm, rwx, false) + } + case posix_acl.ACL_GROUP: + if int64(acl.GetID()) == user.GidNumber || in(group.GroupName, user.Groups) { + updatePerms(perm, rwx, false) + } + case posix_acl.ACL_MASK: + updatePerms(perm, rwx, true) + case posix_acl.ACL_OTHERS: + updatePerms(perm, rwx, false) + } + } + + return +} + +func (fs *cephfs) getFullPermissionSet(ctx context.Context, mount Mount, path string) (permList []*provider.Grant) { + acls := &posix_acl.Acl{} + var xattr []byte + var err error + if xattr, err = mount.GetXattr(path, aclXattr); err != nil { + return nil + } + acls.Decode(xattr) + + for _, acl := range acls.List { + rwx := strings.Split(acl.String(), ":")[2] + switch acl.GetType() { + case posix_acl.ACL_USER: + user, err := fs.getUserByID(ctx, fmt.Sprint(acl.GetID())) + if err != nil { + return nil + } + userGrant := &provider.Grant{ + Grantee: &provider.Grantee{ + Type: provider.GranteeType_GRANTEE_TYPE_USER, + Id: &provider.Grantee_UserId{UserId: user.Id}, + }, + Permissions: &provider.ResourcePermissions{}, + } + updatePerms(userGrant.Permissions, rwx, false) + permList = append(permList, userGrant) + case posix_acl.ACL_GROUP: + group, err := fs.getGroupByID(ctx, fmt.Sprint(acl.GetID())) + if err != nil { + return nil + } + groupGrant := &provider.Grant{ + Grantee: &provider.Grantee{ + Type: provider.GranteeType_GRANTEE_TYPE_GROUP, + Id: &provider.Grantee_GroupId{GroupId: group.Id}, + }, + Permissions: &provider.ResourcePermissions{}, + } + updatePerms(groupGrant.Permissions, rwx, false) + permList = append(permList, groupGrant) + } + } + + return +} + +/* +func permToIntRefl(p *provider.ResourcePermissions) (result uint16) { + if p == nil { return 0b111 } //rwx + + item := reflect.ValueOf(p).Elem() + for _, op := range "rwx" { + for _, perm := range perms[op] { + if item.FieldByName(perm).Bool() { + result |= op2int[op] + break //if value is 1 then bitwise OR can never change it again + } + } + } + + return +} +*/ + +func permToInt(rp *provider.ResourcePermissions) (result uint16) { + if rp == nil { + return 0b111 // rwx + } + if rp.Stat || rp.GetPath || rp.GetQuota || rp.ListGrants || rp.InitiateFileDownload { + result |= 4 + } + if rp.CreateContainer || rp.Move || rp.Delete || rp.InitiateFileUpload || rp.AddGrant || rp.UpdateGrant || + rp.RemoveGrant || rp.DenyGrant || rp.RestoreFileVersion || rp.PurgeRecycle || rp.RestoreRecycleItem { + result |= 2 + } + if rp.ListRecycle || rp.ListContainer || rp.ListFileVersions { + result |= 1 + } + + return +} + +const ( + updateGrant = iota + removeGrant = iota +) + +func (fs *cephfs) changePerms(ctx context.Context, mt Mount, grant *provider.Grant, path string, method int) (err error) { + buf, err := mt.GetXattr(path, aclXattr) + if err != nil { + return + } + acls := &posix_acl.Acl{} + acls.Decode(buf) + var sid posix_acl.AclSID + + switch grant.Grantee.Type { + case provider.GranteeType_GRANTEE_TYPE_USER: + var user *userpb.User + if user, err = fs.getUserByOpaqueID(ctx, grant.Grantee.GetUserId().OpaqueId); err != nil { + return + } + sid.SetUid(uint32(user.UidNumber)) + case provider.GranteeType_GRANTEE_TYPE_GROUP: + var group *grouppb.Group + if group, err = fs.getGroupByOpaqueID(ctx, grant.Grantee.GetGroupId().OpaqueId); err != nil { + return + } + sid.SetGid(uint32(group.GidNumber)) + default: + return errors.New("cephfs: invalid grantee type") + } + + var found = false + var i int + for i = range acls.List { + if acls.List[i].AclSID == sid { + found = true + } + } + + if method == updateGrant { + if found { + acls.List[i].Perm |= permToInt(grant.Permissions) + if acls.List[i].Perm == 0 { // remove empty grant + acls.List = append(acls.List[:i], acls.List[i+1:]...) + } + } else { + acls.List = append(acls.List, posix_acl.AclElement{ + AclSID: sid, + Perm: permToInt(grant.Permissions), + }) + } + } else { //removeGrant + if found { + acls.List[i].Perm &^= permToInt(grant.Permissions) //bitwise and-not, to clear bits on Perm + if acls.List[i].Perm == 0 { // remove empty grant + acls.List = append(acls.List[:i], acls.List[i+1:]...) + } + } + } + + err = mt.SetXattr(path, aclXattr, acls.Encode(), 0) + + return +} + +/* +func updatePermsRefl(rp *provider.ResourcePermissions, acl string, unset bool) { + if rp == nil { return } + for _, t := range "rwx" { + if strings.ContainsRune(acl, t) { + for _, i := range perms[t] { + reflect.ValueOf(rp).Elem().FieldByName(i).SetBool(true) + } + } else if unset { + for _, i := range perms[t] { + reflect.ValueOf(rp).Elem().FieldByName(i).SetBool(false) + } + } + } +} +*/ + +func updatePerms(rp *provider.ResourcePermissions, acl string, unset bool) { + if rp == nil { + return + } + if strings.ContainsRune(acl, 'r') { + rp.Stat = true + rp.GetPath = true + rp.GetQuota = true + rp.InitiateFileDownload = true + rp.ListGrants = true + } else if unset { + rp.Stat = false + rp.GetPath = false + rp.GetQuota = false + rp.InitiateFileDownload = false + rp.ListGrants = false + } + if strings.ContainsRune(acl, 'w') { + rp.AddGrant = true + rp.DenyGrant = true + rp.CreateContainer = true + rp.Delete = true + rp.InitiateFileUpload = true + rp.Move = true + rp.RemoveGrant = true + rp.PurgeRecycle = true + rp.RestoreFileVersion = true + rp.RestoreRecycleItem = true + rp.UpdateGrant = true + } else if unset { + rp.AddGrant = false + rp.DenyGrant = false + rp.CreateContainer = false + rp.Delete = false + rp.InitiateFileUpload = false + rp.Move = false + rp.RemoveGrant = false + rp.PurgeRecycle = false + rp.RestoreFileVersion = false + rp.RestoreRecycleItem = false + rp.UpdateGrant = false + } + if strings.ContainsRune(acl, 'x') { + rp.ListRecycle = true + rp.ListContainer = true + rp.ListFileVersions = true + } else if unset { + rp.ListRecycle = false + rp.ListContainer = false + rp.ListFileVersions = false + } +} diff --git a/pkg/storage/fs/cephfs/unsupported.go b/pkg/storage/fs/cephfs/unsupported.go new file mode 100644 index 0000000000..a337f3f789 --- /dev/null +++ b/pkg/storage/fs/cephfs/unsupported.go @@ -0,0 +1,39 @@ +// Copyright 2018-2021 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +//go:build !ceph +// +build !ceph + +package cephfs + +import ( + "github.com/pkg/errors" + + "github.com/cs3org/reva/pkg/storage" + "github.com/cs3org/reva/pkg/storage/fs/registry" +) + +func init() { + registry.Register("cephfs", New) +} + +// New returns an implementation to of the storage.FS interface that talk to +// a ceph filesystem. +func New(m map[string]interface{}) (storage.FS, error) { + return nil, errors.New("cephfs: revad was compiled without CephFS support") +} diff --git a/pkg/storage/fs/cephfs/upload.go b/pkg/storage/fs/cephfs/upload.go new file mode 100644 index 0000000000..2d8eccf7c9 --- /dev/null +++ b/pkg/storage/fs/cephfs/upload.go @@ -0,0 +1,413 @@ +// Copyright 2018-2021 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +//go:build ceph +// +build ceph + +package cephfs + +import ( + "bytes" + "context" + "encoding/json" + "io" + "os" + "path/filepath" + + cephfs2 "github.com/ceph/go-ceph/cephfs" + userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/cs3org/reva/pkg/appctx" + ctx2 "github.com/cs3org/reva/pkg/ctx" + "github.com/cs3org/reva/pkg/errtypes" + "github.com/cs3org/reva/pkg/utils" + "github.com/google/uuid" + "github.com/pkg/errors" + tusd "github.com/tus/tusd/pkg/handler" +) + +func (fs *cephfs) Upload(ctx context.Context, ref *provider.Reference, r io.ReadCloser) error { + user := fs.makeUser(ctx) + upload, err := fs.GetUpload(ctx, ref.GetPath()) + if err != nil { + metadata := map[string]string{"sizedeferred": "true"} + uploadIDs, err := fs.InitiateUpload(ctx, ref, 0, metadata) + if err != nil { + return err + } + if upload, err = fs.GetUpload(ctx, uploadIDs["simple"]); err != nil { + return errors.Wrap(err, "cephfs: error retrieving upload") + } + } + + uploadInfo := upload.(*fileUpload) + + p := uploadInfo.info.Storage["InternalDestination"] + ok, err := IsChunked(p) + if err != nil { + return errors.Wrap(err, "cephfs: error checking path") + } + if ok { + var assembledFile string + p, assembledFile, err = NewChunkHandler(ctx, fs).WriteChunk(p, r) + if err != nil { + return err + } + if p == "" { + if err = uploadInfo.Terminate(ctx); err != nil { + return errors.Wrap(err, "cephfs: error removing auxiliary files") + } + return errtypes.PartialContent(ref.String()) + } + uploadInfo.info.Storage["InternalDestination"] = p + + user.op(func(cv *cacheVal) { + r, err = cv.mount.Open(assembledFile, os.O_RDONLY, 0) + }) + if err != nil { + return errors.Wrap(err, "cephfs: error opening assembled file") + } + defer r.Close() + defer user.op(func(cv *cacheVal) { + _ = cv.mount.Unlink(assembledFile) + }) + } + + if _, err := uploadInfo.WriteChunk(ctx, 0, r); err != nil { + return errors.Wrap(err, "cephfs: error writing to binary file") + } + + return uploadInfo.FinishUpload(ctx) +} + +func (fs *cephfs) InitiateUpload(ctx context.Context, ref *provider.Reference, uploadLength int64, metadata map[string]string) (map[string]string, error) { + user := fs.makeUser(ctx) + np, err := user.resolveRef(ref) + if err != nil { + return nil, errors.Wrap(err, "cephfs: error resolving reference") + } + + info := tusd.FileInfo{ + MetaData: tusd.MetaData{ + "filename": filepath.Base(np), + "dir": filepath.Dir(np), + }, + Size: uploadLength, + } + + if metadata != nil { + if metadata["mtime"] != "" { + info.MetaData["mtime"] = metadata["mtime"] + } + if _, ok := metadata["sizedeferred"]; ok { + info.SizeIsDeferred = true + } + } + + upload, err := fs.NewUpload(ctx, info) + if err != nil { + return nil, err + } + + info, _ = upload.GetInfo(ctx) + + return map[string]string{ + "simple": info.ID, + "tus": info.ID, + }, nil +} + +// UseIn tells the tus upload middleware which extensions it supports. +func (fs *cephfs) UseIn(composer *tusd.StoreComposer) { + composer.UseCore(fs) + composer.UseTerminater(fs) +} + +func (fs *cephfs) NewUpload(ctx context.Context, info tusd.FileInfo) (upload tusd.Upload, err error) { + log := appctx.GetLogger(ctx) + log.Debug().Interface("info", info).Msg("cephfs: NewUpload") + + user := fs.makeUser(ctx) + + fn := info.MetaData["filename"] + if fn == "" { + return nil, errors.New("cephfs: missing filename in metadata") + } + info.MetaData["filename"] = filepath.Clean(info.MetaData["filename"]) + + dir := info.MetaData["dir"] + if dir == "" { + return nil, errors.New("cephfs: missing dir in metadata") + } + info.MetaData["dir"] = filepath.Clean(info.MetaData["dir"]) + + np := filepath.Join(info.MetaData["dir"], info.MetaData["filename"]) + + info.ID = uuid.New().String() + + binPath := fs.getUploadPath(info.ID) + + info.Storage = map[string]string{ + "Type": "Cephfs", + "BinPath": binPath, + "InternalDestination": np, + + "Idp": user.Id.Idp, + "UserId": user.Id.OpaqueId, + "UserName": user.Username, + "UserType": utils.UserTypeToString(user.Id.Type), + + "LogLevel": log.GetLevel().String(), + } + + // Create binary file with no content + user.op(func(cv *cacheVal) { + var f *cephfs2.File + defer closeFile(f) + f, err = cv.mount.Open(binPath, os.O_CREATE|os.O_WRONLY, filePermDefault) + if err != nil { + return + } + }) + //TODO: if we get two same upload ids, the second one can't upload at all + if err != nil { + return + } + + upload = &fileUpload{ + info: info, + binPath: binPath, + infoPath: binPath + ".info", + fs: fs, + ctx: ctx, + } + + if !info.SizeIsDeferred && info.Size == 0 { + log.Debug().Interface("info", info).Msg("cephfs: finishing upload for empty file") + // no need to create info file and finish directly + err = upload.FinishUpload(ctx) + + return + } + + // writeInfo creates the file by itself if necessary + err = upload.(*fileUpload).writeInfo() + + return +} + +func (fs *cephfs) getUploadPath(uploadID string) string { + return filepath.Join(fs.conf.UploadFolder, uploadID) +} + +// GetUpload returns the Upload for the given upload id +func (fs *cephfs) GetUpload(ctx context.Context, id string) (fup tusd.Upload, err error) { + binPath := fs.getUploadPath(id) + info := tusd.FileInfo{} + if err != nil { + return nil, errtypes.NotFound("bin path for upload " + id + " not found") + } + infoPath := binPath + ".info" + + var data bytes.Buffer + f, err := fs.adminConn.adminMount.Open(infoPath, os.O_RDONLY, 0) + if err != nil { + return + } + _, err = io.Copy(&data, f) + if err != nil { + return + } + if err = json.Unmarshal(data.Bytes(), &info); err != nil { + return + } + + u := &userpb.User{ + Id: &userpb.UserId{ + Idp: info.Storage["Idp"], + OpaqueId: info.Storage["UserId"], + }, + Username: info.Storage["UserName"], + } + ctx = ctx2.ContextSetUser(ctx, u) + user := fs.makeUser(ctx) + + var stat Statx + user.op(func(cv *cacheVal) { + stat, err = cv.mount.Statx(binPath, cephfs2.StatxSize, 0) + }) + if err != nil { + return + } + info.Offset = int64(stat.Size) + + return &fileUpload{ + info: info, + binPath: binPath, + infoPath: infoPath, + fs: fs, + ctx: ctx, + }, nil +} + +type fileUpload struct { + // info stores the current information about the upload + info tusd.FileInfo + // infoPath is the path to the .info file + infoPath string + // binPath is the path to the binary file (which has no extension) + binPath string + // only fs knows how to handle metadata and versions + fs *cephfs + // a context with a user + ctx context.Context +} + +// GetInfo returns the FileInfo +func (upload *fileUpload) GetInfo(ctx context.Context) (tusd.FileInfo, error) { + return upload.info, nil +} + +// GetReader returns an io.Reader for the upload +func (upload *fileUpload) GetReader(ctx context.Context) (file io.Reader, err error) { + user := upload.fs.makeUser(upload.ctx) + user.op(func(cv *cacheVal) { + file, err = cv.mount.Open(upload.binPath, os.O_RDONLY, 0) + }) + return +} + +// WriteChunk writes the stream from the reader to the given offset of the upload +func (upload *fileUpload) WriteChunk(ctx context.Context, offset int64, src io.Reader) (n int64, err error) { + var file io.WriteCloser + user := upload.fs.makeUser(upload.ctx) + user.op(func(cv *cacheVal) { + file, err = cv.mount.Open(upload.binPath, os.O_WRONLY|os.O_APPEND, 0) + }) + if err != nil { + return 0, err + } + defer file.Close() + + n, err = io.Copy(file, src) + + // If the HTTP PATCH request gets interrupted in the middle (e.g. because + // the user wants to pause the upload), Go's net/http returns an io.ErrUnexpectedEOF. + // However, for OwnCloudStore it's not important whether the stream has ended + // on purpose or accidentally. + if err != nil { + if err != io.ErrUnexpectedEOF { + return n, err + } + } + + upload.info.Offset += n + err = upload.writeInfo() + + return n, err +} + +// writeInfo updates the entire information. Everything will be overwritten. +func (upload *fileUpload) writeInfo() error { + data, err := json.Marshal(upload.info) + + if err != nil { + return err + } + user := upload.fs.makeUser(upload.ctx) + user.op(func(cv *cacheVal) { + var file io.WriteCloser + if file, err = cv.mount.Open(upload.infoPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, filePermDefault); err != nil { + return + } + defer file.Close() + + _, err = io.Copy(file, bytes.NewReader(data)) + }) + + return err +} + +// FinishUpload finishes an upload and moves the file to the internal destination +func (upload *fileUpload) FinishUpload(ctx context.Context) (err error) { + + np := upload.info.Storage["InternalDestination"] + + // TODO check etag with If-Match header + // if destination exists + // if _, err := os.Stat(np); err == nil { + // the local storage does not store metadata + // the fileid is based on the path, so no we do not need to copy it to the new file + // the local storage does not track revisions + // } + + // if destination exists + // if _, err := os.Stat(np); err == nil { + // create revision + // if err := upload.fs.archiveRevision(upload.ctx, np); err != nil { + // return err + // } + // } + + user := upload.fs.makeUser(upload.ctx) + log := appctx.GetLogger(ctx) + + user.op(func(cv *cacheVal) { + err = cv.mount.Rename(upload.binPath, np) + }) + if err != nil { + return errors.Wrap(err, upload.binPath) + } + + // only delete the upload if it was successfully written to the fs + user.op(func(cv *cacheVal) { + err = cv.mount.Unlink(upload.infoPath) + }) + if err != nil { + if err.Error() != errNotFound { + log.Err(err).Interface("info", upload.info).Msg("cephfs: could not delete upload metadata") + } + } + + // TODO: set mtime if specified in metadata + + return +} + +// To implement the termination extension as specified in https://tus.io/protocols/resumable-upload.html#termination +// - the storage needs to implement AsTerminatableUpload +// - the upload needs to implement Terminate + +// AsTerminatableUpload returns a a TerminatableUpload +func (fs *cephfs) AsTerminatableUpload(upload tusd.Upload) tusd.TerminatableUpload { + return upload.(*fileUpload) +} + +// Terminate terminates the upload +func (upload *fileUpload) Terminate(ctx context.Context) (err error) { + user := upload.fs.makeUser(upload.ctx) + + user.op(func(cv *cacheVal) { + if err = cv.mount.Unlink(upload.infoPath); err != nil { + return + } + err = cv.mount.Unlink(upload.binPath) + }) + + return +} diff --git a/pkg/storage/fs/cephfs/user.go b/pkg/storage/fs/cephfs/user.go new file mode 100644 index 0000000000..e99686adfd --- /dev/null +++ b/pkg/storage/fs/cephfs/user.go @@ -0,0 +1,242 @@ +// Copyright 2018-2021 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +//go:build ceph +// +build ceph + +package cephfs + +import ( + "context" + "fmt" + "path/filepath" + "strconv" + "strings" + "syscall" + + "github.com/cs3org/reva/pkg/errtypes" + + cephfs2 "github.com/ceph/go-ceph/cephfs" + userv1beta1 "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + typesv1beta1 "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" + ctx2 "github.com/cs3org/reva/pkg/ctx" + "github.com/cs3org/reva/pkg/mime" + "github.com/cs3org/reva/pkg/storage/utils/templates" + "github.com/pkg/errors" +) + +type callBack func(cb *cacheVal) + +// User custom type to add functionality to current struct +type User struct { + *userv1beta1.User + fs *cephfs + ctx context.Context + home string +} + +func (fs *cephfs) makeUser(ctx context.Context) *User { + u := ctx2.ContextMustGetUser(ctx) + home := filepath.Join(fs.conf.Root, templates.WithUser(u, fs.conf.UserLayout)) + return &User{u, fs, ctx, home} +} + +func (user *User) absPath(path string) string { + //shares will always be absolute to avoid prepending the user path to the path of the file's owner + if !filepath.IsAbs(path) { + path = filepath.Join(user.home, path) + } + + return path +} + +func (user *User) op(cb callBack) { + conn := user.fs.conn + if err := conn.lock.Acquire(conn.ctx, 1); err != nil { + return + } + defer conn.lock.Release(1) + + val, found := conn.cache.Get(user.Id.OpaqueId) + if !found { + cvalue := newConn(user) + if cvalue != nil { + conn.cache.Set(user.Id.OpaqueId, cvalue, 1) + } else { + return + } + cb(cvalue) + return + } + + cb(val.(*cacheVal)) +} + +func (user *User) fileAsResourceInfo(cv *cacheVal, path string, stat *cephfs2.CephStatx, mdKeys []string) (ri *provider.ResourceInfo, err error) { + var ( + _type provider.ResourceType + target string + size uint64 + buf []byte + ) + + switch int(stat.Mode) & syscall.S_IFMT { + case syscall.S_IFDIR: + _type = provider.ResourceType_RESOURCE_TYPE_CONTAINER + if buf, err = cv.mount.GetXattr(path, "ceph.dir.rbytes"); err == nil { + size, err = strconv.ParseUint(string(buf), 10, 64) + } + case syscall.S_IFLNK: + _type = provider.ResourceType_RESOURCE_TYPE_SYMLINK + target, err = cv.mount.Readlink(path) + case syscall.S_IFREG: + _type = provider.ResourceType_RESOURCE_TYPE_FILE + size = stat.Size + default: + return nil, errors.New("cephfs: unknown entry type") + } + + if err != nil { + return + } + + var xattrs []string + keys := make(map[string]bool, len(mdKeys)) + for _, key := range mdKeys { + keys[key] = true + } + if keys["*"] || len(keys) == 0 { + mdKeys = []string{} + keys = map[string]bool{} + } + mx := make(map[string]string) + if xattrs, err = cv.mount.ListXattr(path); err == nil { + for _, xattr := range xattrs { + if len(mdKeys) == 0 || keys[xattr] { + if buf, err := cv.mount.GetXattr(path, xattr); err == nil { + mx[xattr] = string(buf) + } + } + } + } + + //TODO(tmourati): Add entry id logic here + + var etag string + if isDir(_type) { + rctime, _ := cv.mount.GetXattr(path, "ceph.dir.rctime") + etag = fmt.Sprint(stat.Inode) + ":" + string(rctime) + } else { + etag = fmt.Sprint(stat.Inode) + ":" + strconv.FormatInt(stat.Ctime.Sec, 10) + } + + mtime := &typesv1beta1.Timestamp{ + Seconds: uint64(stat.Mtime.Sec), + Nanos: uint32(stat.Mtime.Nsec), + } + + perms := getPermissionSet(user, stat, cv.mount, path) + + for key := range mx { + if !strings.HasPrefix(key, xattrUserNs) { + delete(mx, key) + } + } + + var checksum provider.ResourceChecksum + var md5 string + if _type == provider.ResourceType_RESOURCE_TYPE_FILE { + md5tsBA, err := cv.mount.GetXattr(path, xattrMd5ts) //local error inside if scope + if err == nil { + md5ts, _ := strconv.ParseInt(string(md5tsBA), 10, 64) + if stat.Mtime.Sec == md5ts { + md5BA, err := cv.mount.GetXattr(path, xattrMd5) + if err != nil { + md5, err = calcChecksum(path, cv.mount, stat) + } else { + md5 = string(md5BA) + } + } else { + md5, err = calcChecksum(path, cv.mount, stat) + } + } else { + md5, err = calcChecksum(path, cv.mount, stat) + } + + if err != nil && err.Error() == errPermissionDenied { + checksum.Type = provider.ResourceChecksumType_RESOURCE_CHECKSUM_TYPE_UNSET + } else if err != nil { + return nil, errors.New("cephfs: error calculating checksum of file") + } else { + checksum.Type = provider.ResourceChecksumType_RESOURCE_CHECKSUM_TYPE_MD5 + checksum.Sum = md5 + } + } else { + checksum.Type = provider.ResourceChecksumType_RESOURCE_CHECKSUM_TYPE_UNSET + } + + var ownerID *userv1beta1.UserId + if stat.Uid != 0 { + var owner *userv1beta1.User + if int64(stat.Uid) != user.UidNumber { + owner, err = user.fs.getUserByID(user.ctx, fmt.Sprint(stat.Uid)) + } else { + owner = user.User + } + + if owner == nil { + return nil, errors.New("cephfs: error getting owner of entry: " + path) + } + + ownerID = owner.Id + } else { + ownerID = &userv1beta1.UserId{OpaqueId: "root"} + } + + ri = &provider.ResourceInfo{ + Type: _type, + Id: &provider.ResourceId{OpaqueId: fmt.Sprint(stat.Inode)}, + Checksum: &checksum, + Etag: etag, + MimeType: mime.Detect(isDir(_type), path), + Mtime: mtime, + Path: path, + PermissionSet: perms, + Size: size, + Owner: ownerID, + Target: target, + ArbitraryMetadata: &provider.ArbitraryMetadata{Metadata: mx}, + } + + return +} + +func (user *User) resolveRef(ref *provider.Reference) (str string, err error) { + if ref == nil { + return "", fmt.Errorf("cephfs: nil reference") + } + + if str = ref.GetPath(); str == "" { + return "", errtypes.NotSupported("cephfs: entry IDs not currently supported") + } + + str = removeLeadingSlash(str) //path must be relative + + return +} diff --git a/pkg/storage/fs/cephfs/utils.go b/pkg/storage/fs/cephfs/utils.go new file mode 100644 index 0000000000..e25b3bc5fa --- /dev/null +++ b/pkg/storage/fs/cephfs/utils.go @@ -0,0 +1,245 @@ +// Copyright 2018-2021 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +//go:build ceph +// +build ceph + +package cephfs + +import ( + "crypto/md5" + "encoding/hex" + "fmt" + "io" + "os" + "path/filepath" + "strconv" + "strings" + + cephfs2 "github.com/ceph/go-ceph/cephfs" + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" +) + +// Mount type +type Mount = *cephfs2.MountInfo + +// Statx type +type Statx = *cephfs2.CephStatx + +var dirPermFull = uint32(0777) +var dirPermDefault = uint32(0775) +var filePermDefault = uint32(0660) + +func closeDir(directory *cephfs2.Directory) { + if directory != nil { + _ = directory.Close() + } +} + +func closeFile(file *cephfs2.File) { + if file != nil { + _ = file.Close() + } +} + +func destroyCephConn(mt Mount, perm *cephfs2.UserPerm) *cacheVal { + if perm != nil { + perm.Destroy() + } + if mt != nil { + _ = mt.Release() + } + return nil +} + +func deleteFile(mount *cephfs2.MountInfo, path string) { + _ = mount.Unlink(path) +} + +func isDir(t provider.ResourceType) bool { + return t == provider.ResourceType_RESOURCE_TYPE_CONTAINER +} + +func (fs *cephfs) makeFIDPath(fid string) string { + return "" //filepath.Join(fs.conf.EIDFolder, fid) +} + +func (fs *cephfs) makeFID(absolutePath string, inode string) (rid *provider.ResourceId, err error) { + sum := md5.New() + sum.Write([]byte(absolutePath)) + fid := fmt.Sprintf("%s-%s", hex.EncodeToString(sum.Sum(nil)), inode) + rid = &provider.ResourceId{OpaqueId: fid} + + _ = fs.adminConn.adminMount.Link(absolutePath, fs.makeFIDPath(fid)) + _ = fs.adminConn.adminMount.SetXattr(absolutePath, xattrEID, []byte(fid), 0) + + return +} + +func (fs *cephfs) getFIDPath(cv *cacheVal, path string) (fid string, err error) { + var buffer []byte + if buffer, err = cv.mount.GetXattr(path, xattrEID); err != nil { + return + } + + return fs.makeFIDPath(string(buffer)), err +} + +func calcChecksum(filepath string, mt Mount, stat Statx) (checksum string, err error) { + file, err := mt.Open(filepath, os.O_RDONLY, 0) + defer closeFile(file) + if err != nil { + return + } + hash := md5.New() + if _, err = io.Copy(hash, file); err != nil { + return + } + checksum = hex.EncodeToString(hash.Sum(nil)) + // we don't care if they fail, the checksum will just be recalculated if an error happens + _ = mt.SetXattr(filepath, xattrMd5ts, []byte(strconv.FormatInt(stat.Mtime.Sec, 10)), 0) + _ = mt.SetXattr(filepath, xattrMd5, []byte(checksum), 0) + + return +} + +func resolveRevRef(mt Mount, ref *provider.Reference, revKey string) (str string, err error) { + var buf []byte + if ref.GetResourceId() != nil { + str, err = mt.Readlink(filepath.Join(snap, revKey, ref.ResourceId.OpaqueId)) + if err != nil { + return "", fmt.Errorf("cephfs: invalid reference %+v", ref) + } + } else if str = ref.GetPath(); str != "" { + buf, err = mt.GetXattr(str, xattrEID) + if err != nil { + return + } + str, err = mt.Readlink(filepath.Join(snap, revKey, string(buf))) + if err != nil { + return + } + } else { + return "", fmt.Errorf("cephfs: empty reference %+v", ref) + } + + return filepath.Join(snap, revKey, str), err +} + +func removeLeadingSlash(path string) string { + return filepath.Join(".", path) +} + +func addLeadingSlash(path string) string { + return filepath.Join("/", path) +} + +func in(lookup string, list []string) bool { + for _, item := range list { + if item == lookup { + return true + } + } + return false +} + +func pathGenerator(path string, reverse bool, str chan string) { + if reverse { + str <- path + for i := range path { + if path[len(path)-i-1] == filepath.Separator { + str <- path[:len(path)-i-1] + } + } + } else { + for i := range path { + if path[i] == filepath.Separator { + str <- path[:i] + } + } + str <- path + } + + close(str) +} + +func walkPath(path string, f func(string) error, reverse bool) (err error) { + paths := make(chan string) + go pathGenerator(path, reverse, paths) + for path := range paths { + if path == "" { + continue + } + if err = f(path); err != nil && err.Error() != errFileExists && err.Error() != errNotFound { + break + } else { + err = nil + } + } + + return +} + +func (fs *cephfs) writeIndex(oid string, value string) (err error) { + return fs.adminConn.radosIO.WriteFull(oid, []byte(value)) +} + +func (fs *cephfs) removeIndex(oid string) error { + return fs.adminConn.radosIO.Delete(oid) +} + +func (fs *cephfs) resolveIndex(oid string) (fullPath string, err error) { + var i int + var currPath strings.Builder + root := string(filepath.Separator) + offset := uint64(0) + io := fs.adminConn.radosIO + bsize := 4096 + buffer := make([]byte, bsize) + for { + for { //read object + i, err = io.Read(oid, buffer, offset) + offset += uint64(bsize) + currPath.Write(buffer) + if err == nil && i >= bsize { + buffer = buffer[:0] + continue + } else { + offset = 0 + break + } + } + if err != nil { + return + } + + ss := strings.SplitN(currPath.String(), string(filepath.Separator), 2) + if len(ss) != 2 { + if currPath.String() == root { + return + } + + return "", fmt.Errorf("cephfs: entry id is not in the form of \"parentID/entryname\"") + } + parentOID := ss[0] + entryName := ss[1] + fullPath = filepath.Join(entryName, fullPath) + oid = parentOID + currPath.Reset() + } +} diff --git a/pkg/storage/fs/loader/loader.go b/pkg/storage/fs/loader/loader.go index f9ff86af04..cd88c5ddc7 100644 --- a/pkg/storage/fs/loader/loader.go +++ b/pkg/storage/fs/loader/loader.go @@ -20,6 +20,7 @@ package loader import ( // Load core storage filesystem backends. + _ "github.com/cs3org/reva/pkg/storage/fs/cephfs" _ "github.com/cs3org/reva/pkg/storage/fs/eos" _ "github.com/cs3org/reva/pkg/storage/fs/eosgrpc" _ "github.com/cs3org/reva/pkg/storage/fs/eosgrpchome" From 81e8416a29d22bc9457d462eecc69a01b368bbda Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 17 Jan 2022 13:11:02 +0100 Subject: [PATCH 45/58] [Build-deps]: Bump github.com/minio/minio-go/v7 from 7.0.20 to 7.0.21 (#2449) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 367e062000..1d2a89f9f0 100644 --- a/go.mod +++ b/go.mod @@ -44,7 +44,7 @@ require ( github.com/mattn/go-sqlite3 v2.0.3+incompatible github.com/maxymania/go-system v0.0.0-20170110133659-647cc364bf0b github.com/mileusna/useragent v1.0.2 - github.com/minio/minio-go/v7 v7.0.20 + github.com/minio/minio-go/v7 v7.0.21 github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/go-testing-interface v1.14.1 // indirect github.com/mitchellh/mapstructure v1.4.3 diff --git a/go.sum b/go.sum index 0f992f1d5a..fba8360edb 100644 --- a/go.sum +++ b/go.sum @@ -526,8 +526,8 @@ github.com/mileusna/useragent v1.0.2 h1:DgVKtiPnjxlb73z9bCwgdUvU2nQNQ97uhgfO8l9u github.com/mileusna/useragent v1.0.2/go.mod h1:3d8TOmwL/5I8pJjyVDteHtgDGcefrFUX4ccGOMKNYYc= github.com/minio/md5-simd v1.1.0 h1:QPfiOqlZH+Cj9teu0t9b1nTBfPbyTl16Of5MeuShdK4= github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw= -github.com/minio/minio-go/v7 v7.0.20 h1:0+Xt1SkCKDgcx5cmo3UxXcJ37u5Gy+/2i/+eQYqmYJw= -github.com/minio/minio-go/v7 v7.0.20/go.mod h1:ei5JjmxwHaMrgsMrn4U/+Nmg+d8MKS1U2DAn1ou4+Do= +github.com/minio/minio-go/v7 v7.0.21 h1:xrc4BQr1Fa4s5RwY0xfMjPZFJ1bcYBCCHYlngBdWV+k= +github.com/minio/minio-go/v7 v7.0.21/go.mod h1:ei5JjmxwHaMrgsMrn4U/+Nmg+d8MKS1U2DAn1ou4+Do= github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU= github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= From 9c814c6a0e5c198b1bbd14cc8f202a754734c05d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 17 Jan 2022 13:11:23 +0100 Subject: [PATCH 46/58] [Build-deps]: Bump github.com/hashicorp/go-hclog from 1.0.0 to 1.1.0 (#2448) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 1d2a89f9f0..8c28a73665 100644 --- a/go.mod +++ b/go.mod @@ -35,7 +35,7 @@ require ( github.com/google/go-querystring v1.1.0 // indirect github.com/google/uuid v1.3.0 github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 - github.com/hashicorp/go-hclog v1.0.0 + github.com/hashicorp/go-hclog v1.1.0 github.com/hashicorp/go-plugin v1.4.3 github.com/huandu/xstrings v1.3.2 // indirect github.com/imdario/mergo v0.3.12 // indirect diff --git a/go.sum b/go.sum index fba8360edb..fe8056c784 100644 --- a/go.sum +++ b/go.sum @@ -412,8 +412,8 @@ github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2 github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-hclog v0.14.1/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= -github.com/hashicorp/go-hclog v1.0.0 h1:bkKf0BeBXcSYa7f5Fyi9gMuQ8gNsxeiNpZjR6VxNZeo= -github.com/hashicorp/go-hclog v1.0.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= +github.com/hashicorp/go-hclog v1.1.0 h1:QsGcniKx5/LuX2eYoeL+Np3UKYPNaN7YKpTh29h8rbw= +github.com/hashicorp/go-hclog v1.1.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= From 7117894ca302df750f1ca9a500cb13cf31db17f8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 17 Jan 2022 13:11:43 +0100 Subject: [PATCH 47/58] [Build-deps]: Bump github.com/BurntSushi/toml from 0.4.1 to 1.0.0 (#2446) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 8c28a73665..de36d8ed5d 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/cs3org/reva require ( bou.ke/monkey v1.0.2 contrib.go.opencensus.io/exporter/prometheus v0.4.0 - github.com/BurntSushi/toml v0.4.1 + github.com/BurntSushi/toml v1.0.0 github.com/Masterminds/goutils v1.1.1 // indirect github.com/Masterminds/semver v1.5.0 // indirect github.com/Masterminds/sprig v2.22.0+incompatible diff --git a/go.sum b/go.sum index fe8056c784..a88045a2cb 100644 --- a/go.sum +++ b/go.sum @@ -60,8 +60,8 @@ github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBp github.com/Azure/go-ntlmssp v0.0.0-20200615164410-66371956d46c h1:/IBSNwUN8+eKzUzbJPqhK839ygXJ82sde8x3ogr6R28= github.com/Azure/go-ntlmssp v0.0.0-20200615164410-66371956d46c/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/toml v0.4.1 h1:GaI7EiDXDRfa8VshkTj7Fym7ha+y8/XxIgD2okUIjLw= -github.com/BurntSushi/toml v0.4.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= +github.com/BurntSushi/toml v1.0.0 h1:dtDWrepsVPfW9H/4y7dDgFc2MBUSeJhlaDtK13CxFlU= +github.com/BurntSushi/toml v1.0.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= From 4cd33f0c4825f79a1cedb5a562cc847947ebdc4b Mon Sep 17 00:00:00 2001 From: Willy Kloucek <34452982+wkloucek@users.noreply.github.com> Date: Mon, 24 Jan 2022 13:02:31 +0100 Subject: [PATCH 48/58] revert go-sqlite downgrade (#2461) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index de36d8ed5d..fcbb1fa64b 100644 --- a/go.mod +++ b/go.mod @@ -41,7 +41,7 @@ require ( github.com/imdario/mergo v0.3.12 // indirect github.com/jedib0t/go-pretty v4.3.0+incompatible github.com/juliangruber/go-intersect v1.1.0 - github.com/mattn/go-sqlite3 v2.0.3+incompatible + github.com/mattn/go-sqlite3 v1.14.10 github.com/maxymania/go-system v0.0.0-20170110133659-647cc364bf0b github.com/mileusna/useragent v1.0.2 github.com/minio/minio-go/v7 v7.0.21 diff --git a/go.sum b/go.sum index a88045a2cb..7690018b81 100644 --- a/go.sum +++ b/go.sum @@ -513,8 +513,8 @@ github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzp github.com/mattn/go-runewidth v0.0.6/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= -github.com/mattn/go-sqlite3 v2.0.3+incompatible h1:gXHsfypPkaMZrKbD5209QV9jbUTJKjyR5WD3HYQSd+U= -github.com/mattn/go-sqlite3 v2.0.3+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +github.com/mattn/go-sqlite3 v1.14.10 h1:MLn+5bFRlWMGoSRmJour3CL1w/qL96mvipqpwQW/Sfk= +github.com/mattn/go-sqlite3 v1.14.10/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/mattn/go-tty v0.0.3 h1:5OfyWorkyO7xP52Mq7tB36ajHDG5OHrmBGIS/DtakQI= github.com/mattn/go-tty v0.0.3/go.mod h1:ihxohKRERHTVzN+aSVRwACLCeqIoZAWpoICkkvrWyR0= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= From 6f0ac6bb2e01b1600417593648999a52925172d6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Jan 2022 13:02:44 +0100 Subject: [PATCH 49/58] [Build-deps]: Bump github.com/google/go-cmp from 0.5.6 to 0.5.7 (#2466) --- go.mod | 2 +- go.sum | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index fcbb1fa64b..598811e8e0 100644 --- a/go.mod +++ b/go.mod @@ -30,7 +30,7 @@ require ( github.com/golang-jwt/jwt v3.2.2+incompatible github.com/golang/protobuf v1.5.2 github.com/gomodule/redigo v1.8.8 - github.com/google/go-cmp v0.5.6 + github.com/google/go-cmp v0.5.7 github.com/google/go-github v17.0.0+incompatible github.com/google/go-querystring v1.1.0 // indirect github.com/google/uuid v1.3.0 diff --git a/go.sum b/go.sum index 7690018b81..9159089e25 100644 --- a/go.sum +++ b/go.sum @@ -369,8 +369,9 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o= +github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY= github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= From a649b7d2d0468e8f45e1f37a41dbeee45436aec3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Jan 2022 13:03:01 +0100 Subject: [PATCH 50/58] [Build-deps]: Bump github.com/aws/aws-sdk-go from 1.42.27 to 1.42.39 (#2467) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 598811e8e0..6d29c50484 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/Masterminds/sprig v2.22.0+incompatible github.com/ReneKroon/ttlcache/v2 v2.11.0 github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect - github.com/aws/aws-sdk-go v1.42.27 + github.com/aws/aws-sdk-go v1.42.39 github.com/beevik/etree v1.1.0 github.com/bluele/gcache v0.0.2 github.com/c-bata/go-prompt v0.2.5 diff --git a/go.sum b/go.sum index 9159089e25..557492855d 100644 --- a/go.sum +++ b/go.sum @@ -100,8 +100,8 @@ github.com/aws/aws-sdk-go v1.35.24/go.mod h1:tlPOdRjfxPBpNIwqDj61rmsnA85v9jc0Ps9 github.com/aws/aws-sdk-go v1.38.35/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/aws/aws-sdk-go v1.40.11/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q= github.com/aws/aws-sdk-go v1.41.13/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q= -github.com/aws/aws-sdk-go v1.42.27 h1:kxsBXQg3ee6LLbqjp5/oUeDgG7TENFrWYDmEVnd7spU= -github.com/aws/aws-sdk-go v1.42.27/go.mod h1:OGr6lGMAKGlG9CVrYnWYDKIyb829c6EVBRjxqjmPepc= +github.com/aws/aws-sdk-go v1.42.39 h1:6Lso73VoCI8Zmv3zAMv4BNg2gHAKNOlbLv1s/ew90SI= +github.com/aws/aws-sdk-go v1.42.39/go.mod h1:OGr6lGMAKGlG9CVrYnWYDKIyb829c6EVBRjxqjmPepc= github.com/beevik/etree v1.1.0 h1:T0xke/WvNtMoCqgzPhkX2r4rjY3GDZFi+FjpRZY2Jbs= github.com/beevik/etree v1.1.0/go.mod h1:r8Aw8JqVegEf0w2fDnATrX9VpkMcyFeM0FhwO62wh+A= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= From fdad1a529465300cd1bdaabbcae650336a221c4b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Jan 2022 13:03:37 +0100 Subject: [PATCH 51/58] [Build-deps]: Bump github.com/ceph/go-ceph from 0.12.0 to 0.13.0 (#2468) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 6d29c50484..397df4b1f9 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/beevik/etree v1.1.0 github.com/bluele/gcache v0.0.2 github.com/c-bata/go-prompt v0.2.5 - github.com/ceph/go-ceph v0.12.0 + github.com/ceph/go-ceph v0.13.0 github.com/cheggaaa/pb v1.0.29 github.com/coreos/go-oidc v2.2.1+incompatible github.com/cs3org/cato v0.0.0-20200828125504-e418fc54dd5e diff --git a/go.sum b/go.sum index 557492855d..618f451c23 100644 --- a/go.sum +++ b/go.sum @@ -119,8 +119,8 @@ github.com/c-bata/go-prompt v0.2.5 h1:3zg6PecEywxNn0xiqcXHD96fkbxghD+gdB2tbsYfl+ github.com/c-bata/go-prompt v0.2.5/go.mod h1:vFnjEGDIIA/Lib7giyE4E9c50Lvl8j0S+7FVlAwDAVw= github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/ceph/go-ceph v0.12.0 h1:nlFgKQZXOFR4oMnzXsKwTr79Y6EYDwqTrpigICGy/Tw= -github.com/ceph/go-ceph v0.12.0/go.mod h1:mafFpf5Vg8Ai8Bd+FAMvKBHLmtdpTXdRP/TNq8XWegY= +github.com/ceph/go-ceph v0.13.0 h1:69dgIPlNHD2OCz98T0benI4++vcnShGcpQK4RIALjw4= +github.com/ceph/go-ceph v0.13.0/go.mod h1:mafFpf5Vg8Ai8Bd+FAMvKBHLmtdpTXdRP/TNq8XWegY= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= From cf04a9f91b06572557f3f7660a209d6dffc5189b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Jan 2022 13:04:05 +0100 Subject: [PATCH 52/58] [Build-deps]: Bump github.com/onsi/gomega from 1.17.0 to 1.18.0 (#2469) --- go.mod | 4 ++-- go.sum | 9 +++++++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 397df4b1f9..1fe3123044 100644 --- a/go.mod +++ b/go.mod @@ -49,7 +49,7 @@ require ( github.com/mitchellh/go-testing-interface v1.14.1 // indirect github.com/mitchellh/mapstructure v1.4.3 github.com/onsi/ginkgo v1.16.5 - github.com/onsi/gomega v1.17.0 + github.com/onsi/gomega v1.18.0 github.com/pkg/errors v0.9.1 github.com/pkg/xattr v0.4.4 github.com/pquerna/cachecontrol v0.1.0 // indirect @@ -75,7 +75,7 @@ require ( golang.org/x/crypto v0.0.0-20211215165025-cf75a172585e golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c - golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359 + golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e golang.org/x/term v0.0.0-20210916214954-140adaaadfaf google.golang.org/genproto v0.0.0-20211021150943-2b146023228c google.golang.org/grpc v1.42.0 diff --git a/go.sum b/go.sum index 618f451c23..cf019e6049 100644 --- a/go.sum +++ b/go.sum @@ -392,6 +392,7 @@ github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= @@ -575,11 +576,14 @@ github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108 github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= +github.com/onsi/ginkgo/v2 v2.0.0 h1:CcuG/HvWNkkaqCUpJifQY8z7qEMBJya6aLPx6ftGyjQ= +github.com/onsi/ginkgo/v2 v2.0.0/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.17.0 h1:9Luw4uT5HTjHTN8+aNcSThgH1vdXnmdJ8xIfZ4wyTRE= github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= +github.com/onsi/gomega v1.18.0 h1:ngbYoRctxjl8SiF7XgP0NxBFbfHcg3wfHMMaFHWwMTM= +github.com/onsi/gomega v1.18.0/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= @@ -960,8 +964,9 @@ golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210917161153-d61c044b1678/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359 h1:2B5p2L5IfGiD7+b9BOoRMC6DgObAVZV+Fsp050NqXik= golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210916214954-140adaaadfaf h1:Ihq/mm/suC88gF8WFcVwk+OV6Tq+wyA1O0E5UEvDglI= From 393f831f7f44e62d3d230ba2b5242daea71f2ad5 Mon Sep 17 00:00:00 2001 From: David Christofas Date: Mon, 24 Jan 2022 13:09:03 +0100 Subject: [PATCH 53/58] Use permissions API in decomposedfs (#2341) --- .drone.star | 1 + .../unreleased/cs3-permissions-service.md | 5 + cmd/revad/runtime/loader.go | 1 + go.mod | 2 +- go.sum | 4 +- internal/grpc/services/gateway/gateway.go | 1 + internal/grpc/services/gateway/permissions.go | 39 +++++++ internal/grpc/services/loader/loader.go | 1 + .../grpc/services/permissions/permissions.go | 104 ++++++++++++++++++ .../storageprovider/storageprovider.go | 15 +-- pkg/permission/manager/demo/demo.go | 43 ++++++++ pkg/permission/manager/loader/loader.go | 25 +++++ pkg/permission/manager/registry/registry.go | 34 ++++++ pkg/permission/permission.go | 28 +++++ pkg/rgrpc/todo/pool/pool.go | 21 ++++ pkg/storage/fs/nextcloud/nextcloud.go | 2 +- pkg/storage/fs/nextcloud/nextcloud_test.go | 2 +- pkg/storage/fs/owncloud/owncloud.go | 2 +- pkg/storage/fs/owncloudsql/owncloudsql.go | 2 +- pkg/storage/fs/s3/s3.go | 2 +- pkg/storage/storage.go | 2 +- .../utils/decomposedfs/decomposedfs.go | 3 + .../utils/decomposedfs/options/options.go | 2 + pkg/storage/utils/decomposedfs/spaces.go | 45 ++++++-- pkg/storage/utils/eosfs/eosfs.go | 2 +- pkg/storage/utils/localfs/localfs.go | 2 +- tests/oc-integration-tests/drone/gateway.toml | 2 + .../drone/permissions-ocis-ci.toml | 12 ++ .../drone/storage-home-ocis.toml | 2 + .../drone/storage-users-ocis.toml | 2 + tests/oc-integration-tests/local/gateway.toml | 2 + .../local/permissions-ocis-ci.toml | 12 ++ .../local/storage-home.toml | 1 + .../local/storage-users.toml | 1 + 34 files changed, 389 insertions(+), 35 deletions(-) create mode 100644 changelog/unreleased/cs3-permissions-service.md create mode 100644 internal/grpc/services/gateway/permissions.go create mode 100644 internal/grpc/services/permissions/permissions.go create mode 100644 pkg/permission/manager/demo/demo.go create mode 100644 pkg/permission/manager/loader/loader.go create mode 100644 pkg/permission/manager/registry/registry.go create mode 100644 pkg/permission/permission.go create mode 100644 tests/oc-integration-tests/drone/permissions-ocis-ci.toml create mode 100644 tests/oc-integration-tests/local/permissions-ocis-ci.toml diff --git a/.drone.star b/.drone.star index 92460ad3e3..eeed6bd9ba 100644 --- a/.drone.star +++ b/.drone.star @@ -719,6 +719,7 @@ def litmusOcisSpacesDav(): "/drone/src/cmd/revad/revad -c gateway.toml &", "/drone/src/cmd/revad/revad -c storage-home-ocis.toml &", "/drone/src/cmd/revad/revad -c storage-users-ocis.toml &", + "/drone/src/cmd/revad/revad -c permissions-ocis-ci.toml &", "/drone/src/cmd/revad/revad -c users.toml", ], }, diff --git a/changelog/unreleased/cs3-permissions-service.md b/changelog/unreleased/cs3-permissions-service.md new file mode 100644 index 0000000000..b792173e20 --- /dev/null +++ b/changelog/unreleased/cs3-permissions-service.md @@ -0,0 +1,5 @@ +Enhancement: Use CS3 permissions API + +Added calls to the CS3 permissions API to the decomposedfs in order to check the user permissions. + +https://github.com/cs3org/reva/pull/2341 diff --git a/cmd/revad/runtime/loader.go b/cmd/revad/runtime/loader.go index 93f5c68ace..a0df692027 100644 --- a/cmd/revad/runtime/loader.go +++ b/cmd/revad/runtime/loader.go @@ -38,6 +38,7 @@ import ( _ "github.com/cs3org/reva/pkg/ocm/invite/manager/loader" _ "github.com/cs3org/reva/pkg/ocm/provider/authorizer/loader" _ "github.com/cs3org/reva/pkg/ocm/share/manager/loader" + _ "github.com/cs3org/reva/pkg/permission/manager/loader" _ "github.com/cs3org/reva/pkg/publicshare/manager/loader" _ "github.com/cs3org/reva/pkg/rhttp/datatx/manager/loader" _ "github.com/cs3org/reva/pkg/share/cache/loader" diff --git a/go.mod b/go.mod index 1fe3123044..2fdb36d86d 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ require ( github.com/cheggaaa/pb v1.0.29 github.com/coreos/go-oidc v2.2.1+incompatible github.com/cs3org/cato v0.0.0-20200828125504-e418fc54dd5e - github.com/cs3org/go-cs3apis v0.0.0-20211214102047-7ce3134d7bf8 + github.com/cs3org/go-cs3apis v0.0.0-20211214102128-4e8745ab1654 github.com/cubewise-code/go-mime v0.0.0-20200519001935-8c5762b177d8 github.com/dgraph-io/ristretto v0.1.0 github.com/eventials/go-tus v0.0.0-20200718001131-45c7ec8f5d59 diff --git a/go.sum b/go.sum index cf019e6049..ae31f5ef35 100644 --- a/go.sum +++ b/go.sum @@ -136,8 +136,8 @@ github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSV github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cs3org/cato v0.0.0-20200828125504-e418fc54dd5e h1:tqSPWQeueWTKnJVMJffz4pz0o1WuQxJ28+5x5JgaHD8= github.com/cs3org/cato v0.0.0-20200828125504-e418fc54dd5e/go.mod h1:XJEZ3/EQuI3BXTp/6DUzFr850vlxq11I6satRtz0YQ4= -github.com/cs3org/go-cs3apis v0.0.0-20211214102047-7ce3134d7bf8 h1:PqOprF37OvwCbAN5W23znknGk6N/LMayqLAeP904FHE= -github.com/cs3org/go-cs3apis v0.0.0-20211214102047-7ce3134d7bf8/go.mod h1:UXha4TguuB52H14EMoSsCqDj7k8a/t7g4gVP+bgY5LY= +github.com/cs3org/go-cs3apis v0.0.0-20211214102128-4e8745ab1654 h1:ha5tiuuFyDrwKUrVEc3TrRDFgTKVQ9NGDRmEP0PRPno= +github.com/cs3org/go-cs3apis v0.0.0-20211214102128-4e8745ab1654/go.mod h1:UXha4TguuB52H14EMoSsCqDj7k8a/t7g4gVP+bgY5LY= github.com/cubewise-code/go-mime v0.0.0-20200519001935-8c5762b177d8 h1:Z9lwXumT5ACSmJ7WGnFl+OMLLjpz5uR2fyz7dC255FI= github.com/cubewise-code/go-mime v0.0.0-20200519001935-8c5762b177d8/go.mod h1:4abs/jPXcmJzYoYGF91JF9Uq9s/KL5n1jvFDix8KcqY= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= diff --git a/internal/grpc/services/gateway/gateway.go b/internal/grpc/services/gateway/gateway.go index 3a8a6203a5..25819dfb22 100644 --- a/internal/grpc/services/gateway/gateway.go +++ b/internal/grpc/services/gateway/gateway.go @@ -57,6 +57,7 @@ type config struct { GroupProviderEndpoint string `mapstructure:"groupprovidersvc"` DataTxEndpoint string `mapstructure:"datatx"` DataGatewayEndpoint string `mapstructure:"datagateway"` + PermissionsEndpoint string `mapstructure:"permissionssvc"` CommitShareToStorageGrant bool `mapstructure:"commit_share_to_storage_grant"` CommitShareToStorageRef bool `mapstructure:"commit_share_to_storage_ref"` DisableHomeCreationOnLogin bool `mapstructure:"disable_home_creation_on_login"` diff --git a/internal/grpc/services/gateway/permissions.go b/internal/grpc/services/gateway/permissions.go new file mode 100644 index 0000000000..2b1806633a --- /dev/null +++ b/internal/grpc/services/gateway/permissions.go @@ -0,0 +1,39 @@ +// Copyright 2021 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package gateway + +import ( + "context" + + permissions "github.com/cs3org/go-cs3apis/cs3/permissions/v1beta1" + "github.com/cs3org/reva/pkg/rgrpc/status" + "github.com/cs3org/reva/pkg/rgrpc/todo/pool" + "github.com/pkg/errors" +) + +func (s *svc) CheckPermission(ctx context.Context, req *permissions.CheckPermissionRequest) (*permissions.CheckPermissionResponse, error) { + c, err := pool.GetPermissionsClient(s.c.PermissionsEndpoint) + if err != nil { + err = errors.Wrap(err, "gateway: error calling GetPermissionssClient") + return &permissions.CheckPermissionResponse{ + Status: status.NewInternal(ctx, err, "error getting permissions client"), + }, nil + } + return c.CheckPermission(ctx, req) +} diff --git a/internal/grpc/services/loader/loader.go b/internal/grpc/services/loader/loader.go index 118eeed39e..e0161997d2 100644 --- a/internal/grpc/services/loader/loader.go +++ b/internal/grpc/services/loader/loader.go @@ -33,6 +33,7 @@ import ( _ "github.com/cs3org/reva/internal/grpc/services/ocminvitemanager" _ "github.com/cs3org/reva/internal/grpc/services/ocmproviderauthorizer" _ "github.com/cs3org/reva/internal/grpc/services/ocmshareprovider" + _ "github.com/cs3org/reva/internal/grpc/services/permissions" _ "github.com/cs3org/reva/internal/grpc/services/preferences" _ "github.com/cs3org/reva/internal/grpc/services/publicshareprovider" _ "github.com/cs3org/reva/internal/grpc/services/publicstorageprovider" diff --git a/internal/grpc/services/permissions/permissions.go b/internal/grpc/services/permissions/permissions.go new file mode 100644 index 0000000000..4479fdb88c --- /dev/null +++ b/internal/grpc/services/permissions/permissions.go @@ -0,0 +1,104 @@ +// Copyright 2021 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package permissions + +import ( + "context" + "fmt" + + permissions "github.com/cs3org/go-cs3apis/cs3/permissions/v1beta1" + rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" + "github.com/cs3org/reva/pkg/permission" + "github.com/cs3org/reva/pkg/permission/manager/registry" + "github.com/cs3org/reva/pkg/rgrpc" + "github.com/mitchellh/mapstructure" + "github.com/pkg/errors" + "google.golang.org/grpc" +) + +func init() { + rgrpc.Register("permissions", New) +} + +type config struct { + Driver string `mapstructure:"driver" docs:"localhome;The permission driver to be used."` + Drivers map[string]map[string]interface{} `mapstructure:"drivers" docs:"url:pkg/permission/permission.go"` +} + +func parseConfig(m map[string]interface{}) (*config, error) { + c := &config{} + if err := mapstructure.Decode(m, c); err != nil { + err = errors.Wrap(err, "error decoding conf") + return nil, err + } + return c, nil +} + +type service struct { + manager permission.Manager +} + +// New returns a new PermissionsServiceServer +func New(m map[string]interface{}, ss *grpc.Server) (rgrpc.Service, error) { + c, err := parseConfig(m) + if err != nil { + return nil, err + } + + f, ok := registry.NewFuncs[c.Driver] + if !ok { + return nil, fmt.Errorf("could not get permission manager '%s'", c.Driver) + } + manager, err := f(c.Drivers[c.Driver]) + if err != nil { + return nil, err + } + + service := &service{manager: manager} + return service, nil +} + +func (s *service) Close() error { + return nil +} + +func (s *service) UnprotectedEndpoints() []string { + return []string{} +} + +func (s *service) Register(ss *grpc.Server) { + permissions.RegisterPermissionsAPIServer(ss, s) +} + +func (s *service) CheckPermission(ctx context.Context, req *permissions.CheckPermissionRequest) (*permissions.CheckPermissionResponse, error) { + var subject string + switch ref := req.SubjectRef.Spec.(type) { + case *permissions.SubjectReference_UserId: + subject = ref.UserId.OpaqueId + case *permissions.SubjectReference_GroupId: + subject = ref.GroupId.OpaqueId + } + var status *rpc.Status + if ok := s.manager.CheckPermission(req.Permission, subject, req.Ref); ok { + status = &rpc.Status{Code: rpc.Code_CODE_OK} + } else { + status = &rpc.Status{Code: rpc.Code_CODE_PERMISSION_DENIED} + } + return &permissions.CheckPermissionResponse{Status: status}, nil +} diff --git a/internal/grpc/services/storageprovider/storageprovider.go b/internal/grpc/services/storageprovider/storageprovider.go index 13dc51ed15..82e09db0bb 100644 --- a/internal/grpc/services/storageprovider/storageprovider.go +++ b/internal/grpc/services/storageprovider/storageprovider.go @@ -20,7 +20,6 @@ package storageprovider import ( "context" - "encoding/json" "fmt" "net/url" "os" @@ -579,19 +578,7 @@ func hasNodeID(s *provider.StorageSpace) bool { func (s *service) ListStorageSpaces(ctx context.Context, req *provider.ListStorageSpacesRequest) (*provider.ListStorageSpacesResponse, error) { log := appctx.GetLogger(ctx) - // This is just a quick hack to get the users permission into reva. - // Replace this as soon as we have a proper system to check the users permissions. - opaque := req.Opaque - var permissions map[string]struct{} - if opaque != nil { - entry := opaque.Map["permissions"] - err := json.Unmarshal(entry.Value, &permissions) - if err != nil { - return nil, err - } - } - - spaces, err := s.storage.ListStorageSpaces(ctx, req.Filters, permissions) + spaces, err := s.storage.ListStorageSpaces(ctx, req.Filters) if err != nil { var st *rpc.Status switch err.(type) { diff --git a/pkg/permission/manager/demo/demo.go b/pkg/permission/manager/demo/demo.go new file mode 100644 index 0000000000..7bb6c10779 --- /dev/null +++ b/pkg/permission/manager/demo/demo.go @@ -0,0 +1,43 @@ +// Copyright 2021 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package demo + +import ( + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/cs3org/reva/pkg/permission" + "github.com/cs3org/reva/pkg/permission/manager/registry" +) + +func init() { + registry.Register("demo", New) +} + +// New returns a new demo permission manager +func New(c map[string]interface{}) (permission.Manager, error) { + return manager{}, nil +} + +type manager struct { +} + +func (m manager) CheckPermission(permission string, subject string, ref *provider.Reference) bool { + // We can currently return true all the time. + // Once we beginn testing roles we need to somehow check the roles of the users here + return true +} diff --git a/pkg/permission/manager/loader/loader.go b/pkg/permission/manager/loader/loader.go new file mode 100644 index 0000000000..5f0bbc5774 --- /dev/null +++ b/pkg/permission/manager/loader/loader.go @@ -0,0 +1,25 @@ +// Copyright 2021 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package loader + +import ( + // Load permission manager drivers + _ "github.com/cs3org/reva/pkg/permission/manager/demo" + // Add your own here +) diff --git a/pkg/permission/manager/registry/registry.go b/pkg/permission/manager/registry/registry.go new file mode 100644 index 0000000000..26f55bebad --- /dev/null +++ b/pkg/permission/manager/registry/registry.go @@ -0,0 +1,34 @@ +// Copyright 2021 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package registry + +import "github.com/cs3org/reva/pkg/permission" + +// NewFunc is the function that permission managers +// should register at init time. +type NewFunc func(map[string]interface{}) (permission.Manager, error) + +// NewFuncs is a map containing all the registered share managers. +var NewFuncs = map[string]NewFunc{} + +// Register registers a new permission manager new function. +// Not safe for concurrent use. Safe for use from package init. +func Register(name string, f NewFunc) { + NewFuncs[name] = f +} diff --git a/pkg/permission/permission.go b/pkg/permission/permission.go new file mode 100644 index 0000000000..e5e5c76a52 --- /dev/null +++ b/pkg/permission/permission.go @@ -0,0 +1,28 @@ +// Copyright 2021 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package permission + +import ( + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" +) + +// Manager defines the interface for the permission service driver +type Manager interface { + CheckPermission(permission string, subject string, ref *provider.Reference) bool +} diff --git a/pkg/rgrpc/todo/pool/pool.go b/pkg/rgrpc/todo/pool/pool.go index 550d7d7499..b2c94b1219 100644 --- a/pkg/rgrpc/todo/pool/pool.go +++ b/pkg/rgrpc/todo/pool/pool.go @@ -32,6 +32,7 @@ import ( ocmcore "github.com/cs3org/go-cs3apis/cs3/ocm/core/v1beta1" invitepb "github.com/cs3org/go-cs3apis/cs3/ocm/invite/v1beta1" ocmprovider "github.com/cs3org/go-cs3apis/cs3/ocm/provider/v1beta1" + permissions "github.com/cs3org/go-cs3apis/cs3/permissions/v1beta1" preferences "github.com/cs3org/go-cs3apis/cs3/preferences/v1beta1" collaboration "github.com/cs3org/go-cs3apis/cs3/sharing/collaboration/v1beta1" link "github.com/cs3org/go-cs3apis/cs3/sharing/link/v1beta1" @@ -70,6 +71,7 @@ var ( ocmCores = newProvider() publicShareProviders = newProvider() preferencesProviders = newProvider() + permissionsProviders = newProvider() appRegistries = newProvider() appProviders = newProvider() storageRegistries = newProvider() @@ -349,6 +351,25 @@ func GetPreferencesClient(endpoint string) (preferences.PreferencesAPIClient, er return v, nil } +// GetPermissionsClient returns a new PermissionsClient. +func GetPermissionsClient(endpoint string) (permissions.PermissionsAPIClient, error) { + permissionsProviders.m.Lock() + defer permissionsProviders.m.Unlock() + + if c, ok := permissionsProviders.conn[endpoint]; ok { + return c.(permissions.PermissionsAPIClient), nil + } + + conn, err := NewConn(endpoint) + if err != nil { + return nil, err + } + + v := permissions.NewPermissionsAPIClient(conn) + permissionsProviders.conn[endpoint] = v + return v, nil +} + // GetAppRegistryClient returns a new AppRegistryClient. func GetAppRegistryClient(endpoint string) (appregistry.RegistryAPIClient, error) { appRegistries.m.Lock() diff --git a/pkg/storage/fs/nextcloud/nextcloud.go b/pkg/storage/fs/nextcloud/nextcloud.go index 04e5b54912..34e2fe3452 100644 --- a/pkg/storage/fs/nextcloud/nextcloud.go +++ b/pkg/storage/fs/nextcloud/nextcloud.go @@ -783,7 +783,7 @@ func (nc *StorageDriver) Unlock(ctx context.Context, ref *provider.Reference) er } // ListStorageSpaces as defined in the storage.FS interface -func (nc *StorageDriver) ListStorageSpaces(ctx context.Context, f []*provider.ListStorageSpacesRequest_Filter, _ map[string]struct{}) ([]*provider.StorageSpace, error) { +func (nc *StorageDriver) ListStorageSpaces(ctx context.Context, f []*provider.ListStorageSpacesRequest_Filter) ([]*provider.StorageSpace, error) { bodyStr, _ := json.Marshal(f) _, respBody, err := nc.do(ctx, Action{"ListStorageSpaces", string(bodyStr)}) if err != nil { diff --git a/pkg/storage/fs/nextcloud/nextcloud_test.go b/pkg/storage/fs/nextcloud/nextcloud_test.go index b81edd643b..f4565a54b5 100644 --- a/pkg/storage/fs/nextcloud/nextcloud_test.go +++ b/pkg/storage/fs/nextcloud/nextcloud_test.go @@ -987,7 +987,7 @@ var _ = Describe("Nextcloud", func() { }, } filters := []*provider.ListStorageSpacesRequest_Filter{filter1, filter2, filter3} - spaces, err := nc.ListStorageSpaces(ctx, filters, nil) + spaces, err := nc.ListStorageSpaces(ctx, filters) Expect(err).ToNot(HaveOccurred()) Expect(len(spaces)).To(Equal(1)) // https://github.com/cs3org/go-cs3apis/blob/970eec3/cs3/storage/provider/v1beta1/resources.pb.go#L1341-L1366 diff --git a/pkg/storage/fs/owncloud/owncloud.go b/pkg/storage/fs/owncloud/owncloud.go index 5e0a65bce8..c75f82fde1 100644 --- a/pkg/storage/fs/owncloud/owncloud.go +++ b/pkg/storage/fs/owncloud/owncloud.go @@ -2255,7 +2255,7 @@ func (fs *ocfs) RestoreRecycleItem(ctx context.Context, basePath, key, relativeP return fs.propagate(ctx, tgt) } -func (fs *ocfs) ListStorageSpaces(ctx context.Context, filter []*provider.ListStorageSpacesRequest_Filter, _ map[string]struct{}) ([]*provider.StorageSpace, error) { +func (fs *ocfs) ListStorageSpaces(ctx context.Context, filter []*provider.ListStorageSpacesRequest_Filter) ([]*provider.StorageSpace, error) { return nil, errtypes.NotSupported("list storage spaces") } diff --git a/pkg/storage/fs/owncloudsql/owncloudsql.go b/pkg/storage/fs/owncloudsql/owncloudsql.go index 64803fc945..ca38b13afe 100644 --- a/pkg/storage/fs/owncloudsql/owncloudsql.go +++ b/pkg/storage/fs/owncloudsql/owncloudsql.go @@ -1950,7 +1950,7 @@ func (fs *owncloudsqlfs) HashFile(path string) (string, string, string, error) { } } -func (fs *owncloudsqlfs) ListStorageSpaces(ctx context.Context, filter []*provider.ListStorageSpacesRequest_Filter, _ map[string]struct{}) ([]*provider.StorageSpace, error) { +func (fs *owncloudsqlfs) ListStorageSpaces(ctx context.Context, filter []*provider.ListStorageSpacesRequest_Filter) ([]*provider.StorageSpace, error) { // TODO(corby): Implement return nil, errtypes.NotSupported("list storage spaces") } diff --git a/pkg/storage/fs/s3/s3.go b/pkg/storage/fs/s3/s3.go index 063800326e..76799b9047 100644 --- a/pkg/storage/fs/s3/s3.go +++ b/pkg/storage/fs/s3/s3.go @@ -702,7 +702,7 @@ func (fs *s3FS) RestoreRecycleItem(ctx context.Context, basePath, key, relativeP return errtypes.NotSupported("restore recycle") } -func (fs *s3FS) ListStorageSpaces(ctx context.Context, filter []*provider.ListStorageSpacesRequest_Filter, _ map[string]struct{}) ([]*provider.StorageSpace, error) { +func (fs *s3FS) ListStorageSpaces(ctx context.Context, filter []*provider.ListStorageSpacesRequest_Filter) ([]*provider.StorageSpace, error) { return nil, errtypes.NotSupported("list storage spaces") } diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index 4256d10cdf..ae5cba3428 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -62,7 +62,7 @@ type FS interface { GetLock(ctx context.Context, ref *provider.Reference) (*provider.Lock, error) RefreshLock(ctx context.Context, ref *provider.Reference, lock *provider.Lock) error Unlock(ctx context.Context, ref *provider.Reference) error - ListStorageSpaces(ctx context.Context, filter []*provider.ListStorageSpacesRequest_Filter, permissions map[string]struct{}) ([]*provider.StorageSpace, error) + ListStorageSpaces(ctx context.Context, filter []*provider.ListStorageSpacesRequest_Filter) ([]*provider.StorageSpace, error) CreateStorageSpace(ctx context.Context, req *provider.CreateStorageSpaceRequest) (*provider.CreateStorageSpaceResponse, error) UpdateStorageSpace(ctx context.Context, req *provider.UpdateStorageSpaceRequest) (*provider.UpdateStorageSpaceResponse, error) } diff --git a/pkg/storage/utils/decomposedfs/decomposedfs.go b/pkg/storage/utils/decomposedfs/decomposedfs.go index 5e40163e6e..b3ac9e138c 100644 --- a/pkg/storage/utils/decomposedfs/decomposedfs.go +++ b/pkg/storage/utils/decomposedfs/decomposedfs.go @@ -39,6 +39,7 @@ import ( ctxpkg "github.com/cs3org/reva/pkg/ctx" "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/logger" + "github.com/cs3org/reva/pkg/sharedconf" "github.com/cs3org/reva/pkg/storage" "github.com/cs3org/reva/pkg/storage/utils/chunking" "github.com/cs3org/reva/pkg/storage/utils/decomposedfs/node" @@ -101,6 +102,8 @@ func NewDefault(m map[string]interface{}, bs tree.Blobstore) (storage.FS, error) lu.Options = o tp := tree.New(o.Root, o.TreeTimeAccounting, o.TreeSizeAccounting, lu, bs) + + o.GatewayAddr = sharedconf.GetGatewaySVC(o.GatewayAddr) return New(o, lu, p, tp) } diff --git a/pkg/storage/utils/decomposedfs/options/options.go b/pkg/storage/utils/decomposedfs/options/options.go index 181d20f19e..892b185a22 100644 --- a/pkg/storage/utils/decomposedfs/options/options.go +++ b/pkg/storage/utils/decomposedfs/options/options.go @@ -53,6 +53,8 @@ type Options struct { Owner string `mapstructure:"owner"` OwnerIDP string `mapstructure:"owner_idp"` OwnerType string `mapstructure:"owner_type"` + + GatewayAddr string `mapstructure:"gateway_addr"` } // New returns a new Options instance for the given configuration diff --git a/pkg/storage/utils/decomposedfs/spaces.go b/pkg/storage/utils/decomposedfs/spaces.go index b34bb57f65..989ae0c124 100644 --- a/pkg/storage/utils/decomposedfs/spaces.go +++ b/pkg/storage/utils/decomposedfs/spaces.go @@ -28,12 +28,14 @@ import ( "strings" userv1beta1 "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" + permissionsv1beta1 "github.com/cs3org/go-cs3apis/cs3/permissions/v1beta1" v1beta11 "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" types "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" ocsconv "github.com/cs3org/reva/internal/http/services/owncloud/ocs/conversions" "github.com/cs3org/reva/pkg/appctx" ctxpkg "github.com/cs3org/reva/pkg/ctx" + "github.com/cs3org/reva/pkg/rgrpc/todo/pool" "github.com/cs3org/reva/pkg/storage/utils/decomposedfs/node" "github.com/cs3org/reva/pkg/storage/utils/decomposedfs/xattrs" "github.com/cs3org/reva/pkg/utils" @@ -156,7 +158,7 @@ func (fs *Decomposedfs) CreateStorageSpace(ctx context.Context, req *provider.Cr // The list can be filtered by space type or space id. // Spaces are persisted with symlinks in /spaces// pointing to ../../nodes/, the root node of the space // The spaceid is a concatenation of storageid + "!" + nodeid -func (fs *Decomposedfs) ListStorageSpaces(ctx context.Context, filter []*provider.ListStorageSpacesRequest_Filter, permissions map[string]struct{}) ([]*provider.StorageSpace, error) { +func (fs *Decomposedfs) ListStorageSpaces(ctx context.Context, filter []*provider.ListStorageSpacesRequest_Filter) ([]*provider.StorageSpace, error) { // TODO check filters // TODO when a space symlink is broken delete the space for cleanup @@ -200,6 +202,28 @@ func (fs *Decomposedfs) ListStorageSpaces(ctx context.Context, filter []*provide return spaces, nil } + client, err := pool.GetGatewayServiceClient(fs.o.GatewayAddr) + if err != nil { + return nil, err + } + + checkRes, err := client.CheckPermission(ctx, &permissionsv1beta1.CheckPermissionRequest{ + Permission: "list-all-spaces", + SubjectRef: &permissionsv1beta1.SubjectReference{ + Spec: &permissionsv1beta1.SubjectReference_UserId{ + UserId: u.Id, + }, + }, + }) + if err != nil { + return nil, err + } + + canListAllSpaces := false + if checkRes.Status.Code == v1beta11.Code_CODE_OK { + canListAllSpaces = true + } + for i := range matches { // always read link in case storage space id != node id if target, err := os.Readlink(matches[i]); err != nil { @@ -226,7 +250,7 @@ func (fs *Decomposedfs) ListStorageSpaces(ctx context.Context, filter []*provide } // TODO apply more filters - space, err := fs.storageSpaceFromNode(ctx, n, matches[i], spaceType, permissions) + space, err := fs.storageSpaceFromNode(ctx, n, matches[i], spaceType, canListAllSpaces) if err != nil { appctx.GetLogger(ctx).Error().Err(err).Interface("node", n).Msg("could not convert to storage space") continue @@ -329,7 +353,7 @@ func (fs *Decomposedfs) createStorageSpace(ctx context.Context, spaceType, nodeI return nil } -func (fs *Decomposedfs) storageSpaceFromNode(ctx context.Context, node *node.Node, nodePath, spaceType string, permissions map[string]struct{}) (*provider.StorageSpace, error) { +func (fs *Decomposedfs) storageSpaceFromNode(ctx context.Context, node *node.Node, nodePath, spaceType string, canListAllSpaces bool) (*provider.StorageSpace, error) { owner, err := node.Owner() if err != nil { return nil, err @@ -357,13 +381,14 @@ func (fs *Decomposedfs) storageSpaceFromNode(ctx context.Context, node *node.Nod user := ctxpkg.ContextMustGetUser(ctx) // filter out spaces user cannot access (currently based on stat permission) - _, canListAllSpaces := permissions["list-all-spaces"] - p, err := node.ReadUserPermissions(ctx, user) - if err != nil { - return nil, err - } - if !(canListAllSpaces || p.Stat) { - return nil, errors.New("user is not allowed to Stat the space") + if !canListAllSpaces { + p, err := node.ReadUserPermissions(ctx, user) + if err != nil { + return nil, err + } + if !p.Stat { + return nil, errors.New("user is not allowed to Stat the space") + } } space.Owner = &userv1beta1.User{ // FIXME only return a UserID, not a full blown user object diff --git a/pkg/storage/utils/eosfs/eosfs.go b/pkg/storage/utils/eosfs/eosfs.go index 50cdc8dfd9..76034da770 100644 --- a/pkg/storage/utils/eosfs/eosfs.go +++ b/pkg/storage/utils/eosfs/eosfs.go @@ -1628,7 +1628,7 @@ func (fs *eosfs) RestoreRecycleItem(ctx context.Context, basePath, key, relative return fs.c.RestoreDeletedEntry(ctx, auth, key) } -func (fs *eosfs) ListStorageSpaces(ctx context.Context, filter []*provider.ListStorageSpacesRequest_Filter, _ map[string]struct{}) ([]*provider.StorageSpace, error) { +func (fs *eosfs) ListStorageSpaces(ctx context.Context, filter []*provider.ListStorageSpacesRequest_Filter) ([]*provider.StorageSpace, error) { return nil, errtypes.NotSupported("list storage spaces") } diff --git a/pkg/storage/utils/localfs/localfs.go b/pkg/storage/utils/localfs/localfs.go index aa55d27715..4c1a68f225 100644 --- a/pkg/storage/utils/localfs/localfs.go +++ b/pkg/storage/utils/localfs/localfs.go @@ -1286,7 +1286,7 @@ func (fs *localfs) RestoreRecycleItem(ctx context.Context, basePath, key, relati return fs.propagate(ctx, localRestorePath) } -func (fs *localfs) ListStorageSpaces(ctx context.Context, filter []*provider.ListStorageSpacesRequest_Filter, _ map[string]struct{}) ([]*provider.StorageSpace, error) { +func (fs *localfs) ListStorageSpaces(ctx context.Context, filter []*provider.ListStorageSpacesRequest_Filter) ([]*provider.StorageSpace, error) { return nil, errtypes.NotSupported("list storage spaces") } diff --git a/tests/oc-integration-tests/drone/gateway.toml b/tests/oc-integration-tests/drone/gateway.toml index e00e43158d..85c17b20c2 100644 --- a/tests/oc-integration-tests/drone/gateway.toml +++ b/tests/oc-integration-tests/drone/gateway.toml @@ -30,6 +30,8 @@ ocmcoresvc = "localhost:14000" ocmshareprovidersvc = "localhost:14000" ocminvitemanagersvc = "localhost:14000" ocmproviderauthorizersvc = "localhost:14000" +# permissions +permissionssvc = "localhost:10000" # other commit_share_to_storage_grant = true commit_share_to_storage_ref = true diff --git a/tests/oc-integration-tests/drone/permissions-ocis-ci.toml b/tests/oc-integration-tests/drone/permissions-ocis-ci.toml new file mode 100644 index 0000000000..ef025245f3 --- /dev/null +++ b/tests/oc-integration-tests/drone/permissions-ocis-ci.toml @@ -0,0 +1,12 @@ +# This config file will start a reva service that: +# - serves the ocis ci permissions service +[shared] +jwt_secret = "Pive-Fumkiu4" + +[grpc] +address = "0.0.0.0:10000" + +[grpc.services.permissions] +driver = "demo" + +[grpc.services.publicshareprovider.drivers.ocisci] diff --git a/tests/oc-integration-tests/drone/storage-home-ocis.toml b/tests/oc-integration-tests/drone/storage-home-ocis.toml index dbc0748f0a..098aaf7fca 100644 --- a/tests/oc-integration-tests/drone/storage-home-ocis.toml +++ b/tests/oc-integration-tests/drone/storage-home-ocis.toml @@ -23,12 +23,14 @@ mount_id = "123e4567-e89b-12d3-a456-426655440000" expose_data_server = true data_server_url = "http://revad-services:12001/data" enable_home_creation = true +gateway_addr = "0.0.0.0:19000" [grpc.services.storageprovider.drivers.ocis] root = "/drone/src/tmp/reva/data" enable_home = true treetime_accounting = true treesize_accounting = true +gateway_addr = "0.0.0.0:19000" # we have a locally running dataprovider [http] diff --git a/tests/oc-integration-tests/drone/storage-users-ocis.toml b/tests/oc-integration-tests/drone/storage-users-ocis.toml index 2d6ef8fc2d..795ba41d54 100644 --- a/tests/oc-integration-tests/drone/storage-users-ocis.toml +++ b/tests/oc-integration-tests/drone/storage-users-ocis.toml @@ -19,12 +19,14 @@ mount_path = "/users" mount_id = "123e4567-e89b-12d3-a456-426655440000" expose_data_server = true data_server_url = "http://revad-services:11001/data" +gateway_addr = "0.0.0.0:19000" [grpc.services.storageprovider.drivers.ocis] root = "/drone/src/tmp/reva/data" treetime_accounting = true treesize_accounting = true userprovidersvc = "localhost:18000" +gateway_addr = "0.0.0.0:19000" # we have a locally running dataprovider [http] diff --git a/tests/oc-integration-tests/local/gateway.toml b/tests/oc-integration-tests/local/gateway.toml index 6a4f46ee23..39d387d724 100644 --- a/tests/oc-integration-tests/local/gateway.toml +++ b/tests/oc-integration-tests/local/gateway.toml @@ -30,6 +30,8 @@ ocmcoresvc = "localhost:14000" ocmshareprovidersvc = "localhost:14000" ocminvitemanagersvc = "localhost:14000" ocmproviderauthorizersvc = "localhost:14000" +# permissions +permissionssvc = "localhost:10000" # other commit_share_to_storage_grant = true commit_share_to_storage_ref = true diff --git a/tests/oc-integration-tests/local/permissions-ocis-ci.toml b/tests/oc-integration-tests/local/permissions-ocis-ci.toml new file mode 100644 index 0000000000..ef025245f3 --- /dev/null +++ b/tests/oc-integration-tests/local/permissions-ocis-ci.toml @@ -0,0 +1,12 @@ +# This config file will start a reva service that: +# - serves the ocis ci permissions service +[shared] +jwt_secret = "Pive-Fumkiu4" + +[grpc] +address = "0.0.0.0:10000" + +[grpc.services.permissions] +driver = "demo" + +[grpc.services.publicshareprovider.drivers.ocisci] diff --git a/tests/oc-integration-tests/local/storage-home.toml b/tests/oc-integration-tests/local/storage-home.toml index 03b4d6ab6f..cd019d9dff 100644 --- a/tests/oc-integration-tests/local/storage-home.toml +++ b/tests/oc-integration-tests/local/storage-home.toml @@ -30,6 +30,7 @@ root = "/var/tmp/reva/data" enable_home = true treetime_accounting = true treesize_accounting = true +gateway_addr = "0.0.0.0:19000" #user_layout = # do we need owner for users? #owner = 95cb8724-03b2-11eb-a0a6-c33ef8ef53ad diff --git a/tests/oc-integration-tests/local/storage-users.toml b/tests/oc-integration-tests/local/storage-users.toml index d2023fc72a..cb37e05342 100644 --- a/tests/oc-integration-tests/local/storage-users.toml +++ b/tests/oc-integration-tests/local/storage-users.toml @@ -38,3 +38,4 @@ root = "/var/tmp/reva/data" enable_home = false treetime_accounting = true treesize_accounting = true +gateway_addr = "0.0.0.0:19000" From 9ed5f32e966e1e65e58b65fcaadc1fa705169960 Mon Sep 17 00:00:00 2001 From: Amrita <54478846+amrita-shrestha@users.noreply.github.com> Date: Tue, 25 Jan 2022 17:41:03 +0545 Subject: [PATCH 54/58] [tests-only]Bump Core Commit Id (#2451) --- .drone.env | 2 +- .drone.star | 4 +-- .../expected-failures-on-OCIS-storage.md | 31 +++++++++---------- .../expected-failures-on-S3NG-storage.md | 31 +++++++++---------- 4 files changed, 31 insertions(+), 37 deletions(-) diff --git a/.drone.env b/.drone.env index 22bb1ee8a6..d0a052d1f2 100644 --- a/.drone.env +++ b/.drone.env @@ -1,3 +1,3 @@ # The test runner source for API tests -CORE_COMMITID=88dc1fc01b8afd254d59cbface6ef48670848e22 +CORE_COMMITID=a91d8d0f045c512aa76ecc92372a0b5ae7b4e3cd CORE_BRANCH=master diff --git a/.drone.star b/.drone.star index eeed6bd9ba..be7b562a6e 100644 --- a/.drone.star +++ b/.drone.star @@ -808,7 +808,7 @@ def ocisIntegrationTests(parallelRuns, skipExceptParts = []): "REVA_LDAP_HOSTNAME": "ldap", "TEST_REVA": "true", "SEND_SCENARIO_LINE_REFERENCES": "true", - "BEHAT_FILTER_TAGS": "~@notToImplementOnOCIS&&~@toImplementOnOCIS&&~comments-app-required&&~@federation-app-required&&~@notifications-app-required&&~systemtags-app-required&&~@provisioning_api-app-required&&~@preview-extension-required&&~@local_storage&&~@skipOnOcis-OCIS-Storage&&~@skipOnOcis", + "BEHAT_FILTER_TAGS": "~@notToImplementOnOCIS&&~@toImplementOnOCIS&&~comments-app-required&&~@federation-app-required&&~@notifications-app-required&&~systemtags-app-required&&~@provisioning_api-app-required&&~@preview-extension-required&&~@local_storage&&~@skipOnOcis-OCIS-Storage&&~@skipOnOcis&&~@personalSpace&&~@issue-ocis-3023", "DIVIDE_INTO_NUM_PARTS": parallelRuns, "RUN_PART": runPart, "EXPECTED_FAILURES_FILE": "/drone/src/tests/acceptance/expected-failures-on-OCIS-storage.md", @@ -883,7 +883,7 @@ def s3ngIntegrationTests(parallelRuns, skipExceptParts = []): "REVA_LDAP_HOSTNAME": "ldap", "TEST_REVA": "true", "SEND_SCENARIO_LINE_REFERENCES": "true", - "BEHAT_FILTER_TAGS": "~@notToImplementOnOCIS&&~@toImplementOnOCIS&&~comments-app-required&&~@federation-app-required&&~@notifications-app-required&&~systemtags-app-required&&~@provisioning_api-app-required&&~@preview-extension-required&&~@local_storage&&~@skipOnOcis-OCIS-Storage&&~@skipOnOcis", + "BEHAT_FILTER_TAGS": "~@notToImplementOnOCIS&&~@toImplementOnOCIS&&~comments-app-required&&~@federation-app-required&&~@notifications-app-required&&~systemtags-app-required&&~@provisioning_api-app-required&&~@preview-extension-required&&~@local_storage&&~@skipOnOcis-OCIS-Storage&&~@skipOnOcis&&~@personalSpace&&~@issue-ocis-3023", "DIVIDE_INTO_NUM_PARTS": parallelRuns, "RUN_PART": runPart, "EXPECTED_FAILURES_FILE": "/drone/src/tests/acceptance/expected-failures-on-S3NG-storage.md", diff --git a/tests/acceptance/expected-failures-on-OCIS-storage.md b/tests/acceptance/expected-failures-on-OCIS-storage.md index cf97c7a7b2..aea236385d 100644 --- a/tests/acceptance/expected-failures-on-OCIS-storage.md +++ b/tests/acceptance/expected-failures-on-OCIS-storage.md @@ -42,7 +42,7 @@ _ocdav: double check the webdav property parsing when custom namespaces are used Synchronization features like etag propagation, setting mtime and locking files #### [Uploading an old method chunked file with checksum should fail using new DAV path](https://github.com/owncloud/ocis/issues/2323) -- [apiMain/checksums.feature:263](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiMain/checksums.feature#L263) +- [apiMain/checksums.feature:369](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiMain/checksums.feature#L369) #### [Webdav LOCK operations](https://github.com/owncloud/ocis/issues/1284) - [apiWebdavLocks/exclusiveLocks.feature:18](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavLocks/exclusiveLocks.feature#L18) @@ -975,20 +975,20 @@ Scenario Outline: search for entry with emoji by pattern - [apiWebdavOperations/search.feature:255](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavOperations/search.feature#L255) Scenario: search for entries across various folders by tags using REPORT method And other missing implementation of favorites -- [apiFavorites/favorites.feature:128](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favorites.feature#L128) -- [apiFavorites/favorites.feature:129](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favorites.feature#L129) -- [apiFavorites/favorites.feature:148](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favorites.feature#L148) -- [apiFavorites/favorites.feature:149](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favorites.feature#L149) -- [apiFavorites/favorites.feature:176](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favorites.feature#L176) -- [apiFavorites/favorites.feature:177](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favorites.feature#L177) +- [apiFavorites/favorites.feature:158](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favorites.feature#L158) +- [apiFavorites/favorites.feature:159](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favorites.feature#L159) +- [apiFavorites/favorites.feature:183](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favorites.feature#L183) +- [apiFavorites/favorites.feature:184](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favorites.feature#L184) +- [apiFavorites/favorites.feature:216](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favorites.feature#L216) +- [apiFavorites/favorites.feature:217](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favorites.feature#L217) - [apiFavorites/favoritesSharingToShares.feature:21](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favoritesSharingToShares.feature#L21) - [apiFavorites/favoritesSharingToShares.feature:22](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favoritesSharingToShares.feature#L22) -- [apiFavorites/favoritesSharingToShares.feature:35](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favoritesSharingToShares.feature#L35) -- [apiFavorites/favoritesSharingToShares.feature:36](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favoritesSharingToShares.feature#L36) -- [apiFavorites/favoritesSharingToShares.feature:48](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favoritesSharingToShares.feature#L48) -- [apiFavorites/favoritesSharingToShares.feature:49](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favoritesSharingToShares.feature#L49) -- [apiFavorites/favoritesSharingToShares.feature:62](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favoritesSharingToShares.feature#L62) -- [apiFavorites/favoritesSharingToShares.feature:63](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favoritesSharingToShares.feature#L63) +- [apiFavorites/favoritesSharingToShares.feature:40](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favoritesSharingToShares.feature#L40) +- [apiFavorites/favoritesSharingToShares.feature:41](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favoritesSharingToShares.feature#L41) +- [apiFavorites/favoritesSharingToShares.feature:58](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favoritesSharingToShares.feature#L58) +- [apiFavorites/favoritesSharingToShares.feature:59](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favoritesSharingToShares.feature#L59) +- [apiFavorites/favoritesSharingToShares.feature:77](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favoritesSharingToShares.feature#L77) +- [apiFavorites/favoritesSharingToShares.feature:78](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favoritesSharingToShares.feature#L78) #### [WWW-Authenticate header for unauthenticated requests is not clear](https://github.com/owncloud/ocis/issues/2285) Scenario Outline: Unauthenticated call @@ -1375,7 +1375,4 @@ _ocs: api compatibility, return correct status code_ ### [Content-type is not multipart/byteranges when downloading file with Range Header](https://github.com/owncloud/ocis/issues/2677) - [apiWebdavOperations/downloadFile.feature:169](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavOperations/downloadFile.feature#L169) -- [apiWebdavOperations/downloadFile.feature:170](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavOperations/downloadFile.feature#L170) - -### [Calculate personal space id in tests when running on reva](https://github.com/owncloud/core/issues/39617) -- [apiWebdavUpload1/uploadFile.feature:32](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavUpload1/uploadFile.feature#L32) +- [apiWebdavOperations/downloadFile.feature:170](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavOperations/downloadFile.feature#L170) \ No newline at end of file diff --git a/tests/acceptance/expected-failures-on-S3NG-storage.md b/tests/acceptance/expected-failures-on-S3NG-storage.md index 4fc1c4446a..18631fd5a5 100644 --- a/tests/acceptance/expected-failures-on-S3NG-storage.md +++ b/tests/acceptance/expected-failures-on-S3NG-storage.md @@ -56,7 +56,7 @@ _ocdav: double check the webdav property parsing when custom namespaces are used Synchronization features like etag propagation, setting mtime and locking files #### [Uploading an old method chunked file with checksum should fail using new DAV path](https://github.com/owncloud/ocis/issues/2323) -- [apiMain/checksums.feature:263](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiMain/checksums.feature#L263) +- [apiMain/checksums.feature:369](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiMain/checksums.feature#L369) #### [Webdav LOCK operations](https://github.com/owncloud/ocis/issues/1284) - [apiWebdavLocks/exclusiveLocks.feature:18](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavLocks/exclusiveLocks.feature#L18) @@ -990,20 +990,20 @@ Scenario Outline: search for entry with emoji by pattern - [apiWebdavOperations/search.feature:255](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavOperations/search.feature#L255) Scenario: search for entries across various folders by tags using REPORT method And other missing implementation of favorites -- [apiFavorites/favorites.feature:128](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favorites.feature#L128) -- [apiFavorites/favorites.feature:129](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favorites.feature#L129) -- [apiFavorites/favorites.feature:148](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favorites.feature#L148) -- [apiFavorites/favorites.feature:149](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favorites.feature#L149) -- [apiFavorites/favorites.feature:176](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favorites.feature#L176) -- [apiFavorites/favorites.feature:177](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favorites.feature#L177) +- [apiFavorites/favorites.feature:158](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favorites.feature#L158) +- [apiFavorites/favorites.feature:159](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favorites.feature#L159) +- [apiFavorites/favorites.feature:183](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favorites.feature#L183) +- [apiFavorites/favorites.feature:184](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favorites.feature#L184) +- [apiFavorites/favorites.feature:216](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favorites.feature#L216) +- [apiFavorites/favorites.feature:217](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favorites.feature#L217) - [apiFavorites/favoritesSharingToShares.feature:21](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favoritesSharingToShares.feature#L21) - [apiFavorites/favoritesSharingToShares.feature:22](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favoritesSharingToShares.feature#L22) -- [apiFavorites/favoritesSharingToShares.feature:35](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favoritesSharingToShares.feature#L35) -- [apiFavorites/favoritesSharingToShares.feature:36](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favoritesSharingToShares.feature#L36) -- [apiFavorites/favoritesSharingToShares.feature:48](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favoritesSharingToShares.feature#L48) -- [apiFavorites/favoritesSharingToShares.feature:49](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favoritesSharingToShares.feature#L49) -- [apiFavorites/favoritesSharingToShares.feature:62](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favoritesSharingToShares.feature#L62) -- [apiFavorites/favoritesSharingToShares.feature:63](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favoritesSharingToShares.feature#L63) +- [apiFavorites/favoritesSharingToShares.feature:40](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favoritesSharingToShares.feature#L40) +- [apiFavorites/favoritesSharingToShares.feature:41](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favoritesSharingToShares.feature#L41) +- [apiFavorites/favoritesSharingToShares.feature:58](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favoritesSharingToShares.feature#L58) +- [apiFavorites/favoritesSharingToShares.feature:59](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favoritesSharingToShares.feature#L59) +- [apiFavorites/favoritesSharingToShares.feature:77](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favoritesSharingToShares.feature#L77) +- [apiFavorites/favoritesSharingToShares.feature:78](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favoritesSharingToShares.feature#L78) #### [WWW-Authenticate header for unauthenticated requests is not clear](https://github.com/owncloud/ocis/issues/2285) Scenario Outline: Unauthenticated call @@ -1375,7 +1375,4 @@ _ocs: api compatibility, return correct status code_ ### [Content-type is not multipart/byteranges when downloading file with Range Header](https://github.com/owncloud/ocis/issues/2677) - [apiWebdavOperations/downloadFile.feature:169](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavOperations/downloadFile.feature#L169) -- [apiWebdavOperations/downloadFile.feature:170](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavOperations/downloadFile.feature#L170) - -### [Calculate personal space id in tests when running on reva](https://github.com/owncloud/core/issues/39617) -- [apiWebdavUpload1/uploadFile.feature:32](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavUpload1/uploadFile.feature#L32) +- [apiWebdavOperations/downloadFile.feature:170](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavOperations/downloadFile.feature#L170) \ No newline at end of file From 0e8c017e2767273e50dfebd4fb61154ec0bb53d8 Mon Sep 17 00:00:00 2001 From: Phil Davis Date: Wed, 26 Jan 2022 08:50:24 +0545 Subject: [PATCH 55/58] Bump commit id for tests 2022-01-25 (#2474) --- .drone.env | 2 +- .../expected-failures-on-OCIS-storage.md | 20 +++++++++---------- .../expected-failures-on-S3NG-storage.md | 20 +++++++++---------- 3 files changed, 21 insertions(+), 21 deletions(-) diff --git a/.drone.env b/.drone.env index d0a052d1f2..a336b97fad 100644 --- a/.drone.env +++ b/.drone.env @@ -1,3 +1,3 @@ # The test runner source for API tests -CORE_COMMITID=a91d8d0f045c512aa76ecc92372a0b5ae7b4e3cd +CORE_COMMITID=a8b0dd3d164da66e060290e23b278817d444d15d CORE_BRANCH=master diff --git a/tests/acceptance/expected-failures-on-OCIS-storage.md b/tests/acceptance/expected-failures-on-OCIS-storage.md index aea236385d..009c3da2a2 100644 --- a/tests/acceptance/expected-failures-on-OCIS-storage.md +++ b/tests/acceptance/expected-failures-on-OCIS-storage.md @@ -607,16 +607,16 @@ Scenario Outline: Retrieving folder quota when quota is set and a file was recei #### [changing user quota gives ocs status 103 / Cannot set quota](https://github.com/owncloud/product/issues/247) _requires a [CS3 user provisioning api that can update the quota for a user](https://github.com/cs3org/cs3apis/pull/95#issuecomment-772780683)_ -- [apiShareOperationsToShares2/uploadToShare.feature:168](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiShareOperationsToShares2/uploadToShare.feature#L168) -- [apiShareOperationsToShares2/uploadToShare.feature:169](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiShareOperationsToShares2/uploadToShare.feature#L169) -- [apiShareOperationsToShares2/uploadToShare.feature:188](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiShareOperationsToShares2/uploadToShare.feature#L188) -- [apiShareOperationsToShares2/uploadToShare.feature:189](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiShareOperationsToShares2/uploadToShare.feature#L189) -- [apiShareOperationsToShares2/uploadToShare.feature:210](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiShareOperationsToShares2/uploadToShare.feature#L210) -- [apiShareOperationsToShares2/uploadToShare.feature:211](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiShareOperationsToShares2/uploadToShare.feature#L211) -- [apiShareOperationsToShares2/uploadToShare.feature:230](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiShareOperationsToShares2/uploadToShare.feature#L230) -- [apiShareOperationsToShares2/uploadToShare.feature:231](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiShareOperationsToShares2/uploadToShare.feature#L231) -- [apiShareOperationsToShares2/uploadToShare.feature:252](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiShareOperationsToShares2/uploadToShare.feature#L252) -- [apiShareOperationsToShares2/uploadToShare.feature:253](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiShareOperationsToShares2/uploadToShare.feature#L253) +- [apiShareOperationsToShares2/uploadToShare.feature:193](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiShareOperationsToShares2/uploadToShare.feature#L193) +- [apiShareOperationsToShares2/uploadToShare.feature:194](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiShareOperationsToShares2/uploadToShare.feature#L194) +- [apiShareOperationsToShares2/uploadToShare.feature:218](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiShareOperationsToShares2/uploadToShare.feature#L218) +- [apiShareOperationsToShares2/uploadToShare.feature:219](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiShareOperationsToShares2/uploadToShare.feature#L219) +- [apiShareOperationsToShares2/uploadToShare.feature:245](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiShareOperationsToShares2/uploadToShare.feature#L245) +- [apiShareOperationsToShares2/uploadToShare.feature:246](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiShareOperationsToShares2/uploadToShare.feature#L246) +- [apiShareOperationsToShares2/uploadToShare.feature:270](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiShareOperationsToShares2/uploadToShare.feature#L270) +- [apiShareOperationsToShares2/uploadToShare.feature:271](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiShareOperationsToShares2/uploadToShare.feature#L271) +- [apiShareOperationsToShares2/uploadToShare.feature:297](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiShareOperationsToShares2/uploadToShare.feature#L297) +- [apiShareOperationsToShares2/uploadToShare.feature:298](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiShareOperationsToShares2/uploadToShare.feature#L298) #### [not possible to move file into a received folder](https://github.com/owncloud/ocis/issues/764) - [apiShareOperationsToShares1/changingFilesShare.feature:24](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiShareOperationsToShares1/changingFilesShare.feature#L24) diff --git a/tests/acceptance/expected-failures-on-S3NG-storage.md b/tests/acceptance/expected-failures-on-S3NG-storage.md index 18631fd5a5..5991fa4bf8 100644 --- a/tests/acceptance/expected-failures-on-S3NG-storage.md +++ b/tests/acceptance/expected-failures-on-S3NG-storage.md @@ -596,16 +596,16 @@ Scenario Outline: Retrieving folder quota when quota is set and a file was recei #### [changing user quota gives ocs status 103 / Cannot set quota](https://github.com/owncloud/product/issues/247) _requires a [CS3 user provisioning api that can update the quota for a user](https://github.com/cs3org/cs3apis/pull/95#issuecomment-772780683)_ -- [apiShareOperationsToShares2/uploadToShare.feature:168](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiShareOperationsToShares2/uploadToShare.feature#L168) -- [apiShareOperationsToShares2/uploadToShare.feature:169](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiShareOperationsToShares2/uploadToShare.feature#L169) -- [apiShareOperationsToShares2/uploadToShare.feature:188](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiShareOperationsToShares2/uploadToShare.feature#L188) -- [apiShareOperationsToShares2/uploadToShare.feature:189](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiShareOperationsToShares2/uploadToShare.feature#L189) -- [apiShareOperationsToShares2/uploadToShare.feature:210](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiShareOperationsToShares2/uploadToShare.feature#L210) -- [apiShareOperationsToShares2/uploadToShare.feature:211](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiShareOperationsToShares2/uploadToShare.feature#L211) -- [apiShareOperationsToShares2/uploadToShare.feature:230](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiShareOperationsToShares2/uploadToShare.feature#L230) -- [apiShareOperationsToShares2/uploadToShare.feature:231](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiShareOperationsToShares2/uploadToShare.feature#L231) -- [apiShareOperationsToShares2/uploadToShare.feature:252](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiShareOperationsToShares2/uploadToShare.feature#L252) -- [apiShareOperationsToShares2/uploadToShare.feature:253](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiShareOperationsToShares2/uploadToShare.feature#L253) +- [apiShareOperationsToShares2/uploadToShare.feature:193](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiShareOperationsToShares2/uploadToShare.feature#L193) +- [apiShareOperationsToShares2/uploadToShare.feature:194](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiShareOperationsToShares2/uploadToShare.feature#L194) +- [apiShareOperationsToShares2/uploadToShare.feature:218](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiShareOperationsToShares2/uploadToShare.feature#L218) +- [apiShareOperationsToShares2/uploadToShare.feature:219](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiShareOperationsToShares2/uploadToShare.feature#L219) +- [apiShareOperationsToShares2/uploadToShare.feature:245](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiShareOperationsToShares2/uploadToShare.feature#L245) +- [apiShareOperationsToShares2/uploadToShare.feature:246](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiShareOperationsToShares2/uploadToShare.feature#L246) +- [apiShareOperationsToShares2/uploadToShare.feature:270](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiShareOperationsToShares2/uploadToShare.feature#L270) +- [apiShareOperationsToShares2/uploadToShare.feature:271](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiShareOperationsToShares2/uploadToShare.feature#L271) +- [apiShareOperationsToShares2/uploadToShare.feature:297](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiShareOperationsToShares2/uploadToShare.feature#L297) +- [apiShareOperationsToShares2/uploadToShare.feature:298](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiShareOperationsToShares2/uploadToShare.feature#L298) #### [not possible to move file into a received folder](https://github.com/owncloud/ocis/issues/764) From b3f53a14b3e4655d01510085a3914a96ff17f696 Mon Sep 17 00:00:00 2001 From: Amrita <54478846+amrita-shrestha@users.noreply.github.com> Date: Wed, 26 Jan 2022 16:21:20 +0545 Subject: [PATCH 56/58] Bump commit id for issue-ocis-3030 (#2476) --- .drone.env | 2 +- .../expected-failures-on-OCIS-storage.md | 35 +++++++++++-------- .../expected-failures-on-S3NG-storage.md | 35 +++++++++++-------- 3 files changed, 43 insertions(+), 29 deletions(-) diff --git a/.drone.env b/.drone.env index a336b97fad..d0afb183e2 100644 --- a/.drone.env +++ b/.drone.env @@ -1,3 +1,3 @@ # The test runner source for API tests -CORE_COMMITID=a8b0dd3d164da66e060290e23b278817d444d15d +CORE_COMMITID=65a12d1858c0708cb1ca2ad620e139e0d33b73ae CORE_BRANCH=master diff --git a/tests/acceptance/expected-failures-on-OCIS-storage.md b/tests/acceptance/expected-failures-on-OCIS-storage.md index 009c3da2a2..3f089be08d 100644 --- a/tests/acceptance/expected-failures-on-OCIS-storage.md +++ b/tests/acceptance/expected-failures-on-OCIS-storage.md @@ -914,26 +914,26 @@ API, search, favorites, config, capabilities, not existing endpoints, CORS and o #### [Trying to access another user's file gives http 403 instead of 404](https://github.com/owncloud/ocis/issues/2175) _ocdav: api compatibility, return correct status code_ -- [apiAuthWebDav/webDavDELETEAuth.feature:38](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiAuthWebDav/webDavDELETEAuth.feature#L38) Scenario: send DELETE requests to another user's webDav endpoints as normal user -- [apiAuthWebDav/webDavPROPFINDAuth.feature:39](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiAuthWebDav/webDavPROPFINDAuth.feature#L39) Scenario: send PROPFIND requests to another user's webDav endpoints as normal user -- [apiAuthWebDav/webDavPROPPATCHAuth.feature:40](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiAuthWebDav/webDavPROPPATCHAuth.feature#L40) Scenario: send PROPPATCH requests to another user's webDav endpoints as normal user -- [apiAuthWebDav/webDavMKCOLAuth.feature:36](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiAuthWebDav/webDavMKCOLAuth.feature#L36) Scenario: send MKCOL requests to another user's webDav endpoints as normal user +- [apiAuthWebDav/webDavDELETEAuth.feature:58](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiAuthWebDav/webDavDELETEAuth.feature#L58) Scenario: send DELETE requests to another user's webDav endpoints as normal user +- [apiAuthWebDav/webDavPROPFINDAuth.feature:57](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiAuthWebDav/webDavPROPFINDAuth.feature#L57) Scenario: send PROPFIND requests to another user's webDav endpoints as normal user +- [apiAuthWebDav/webDavPROPPATCHAuth.feature:58](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiAuthWebDav/webDavPROPPATCHAuth.feature#L58) Scenario: send PROPPATCH requests to another user's webDav endpoints as normal user +- [apiAuthWebDav/webDavMKCOLAuth.feature:54](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiAuthWebDav/webDavMKCOLAuth.feature#L54) Scenario: send MKCOL requests to another user's webDav endpoints as normal user #### [trying to lock file of another user gives http 200](https://github.com/owncloud/ocis/issues/2176) -- [apiAuthWebDav/webDavLOCKAuth.feature:40](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiAuthWebDav/webDavLOCKAuth.feature#L40) Scenario: send LOCK requests to another user's webDav endpoints as normal user +- [apiAuthWebDav/webDavLOCKAuth.feature:58](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiAuthWebDav/webDavLOCKAuth.feature#L58) Scenario: send LOCK requests to another user's webDav endpoints as normal user #### [Renaming a resource to banned name is allowed](https://github.com/owncloud/ocis/issues/1295) _ocdav: api compatibility, return correct status code_ -- [apiAuthWebDav/webDavMOVEAuth.feature:39](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiAuthWebDav/webDavMOVEAuth.feature#L39) Scenario: send MOVE requests to another user's webDav endpoints as normal user +- [apiAuthWebDav/webDavMOVEAuth.feature:57](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiAuthWebDav/webDavMOVEAuth.feature#L57) Scenario: send MOVE requests to another user's webDav endpoints as normal user #### [send POST requests to another user's webDav endpoints as normal user](https://github.com/owncloud/ocis/issues/1287) _ocdav: api compatibility, return correct status code_ -- [apiAuthWebDav/webDavPOSTAuth.feature:40](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiAuthWebDav/webDavPOSTAuth.feature#L40) Scenario: send POST requests to another user's webDav endpoints as normal user +- [apiAuthWebDav/webDavPOSTAuth.feature:58](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiAuthWebDav/webDavPOSTAuth.feature#L58) Scenario: send POST requests to another user's webDav endpoints as normal user #### [Using double slash in URL to access a folder gives 501 and other status codes](https://github.com/owncloud/ocis/issues/1667) -- [apiAuthWebDav/webDavSpecialURLs.feature:24](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiAuthWebDav/webDavSpecialURLs.feature#L24) -- [apiAuthWebDav/webDavSpecialURLs.feature:69](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiAuthWebDav/webDavSpecialURLs.feature#L69) -- [apiAuthWebDav/webDavSpecialURLs.feature:91](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiAuthWebDav/webDavSpecialURLs.feature#L91) +- [apiAuthWebDav/webDavSpecialURLs.feature:34](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiAuthWebDav/webDavSpecialURLs.feature#L34) +- [apiAuthWebDav/webDavSpecialURLs.feature:121](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiAuthWebDav/webDavSpecialURLs.feature#L121) +- [apiAuthWebDav/webDavSpecialURLs.feature:163](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiAuthWebDav/webDavSpecialURLs.feature#L163) #### [Default capabilities for normal user not same as in oC-core](https://github.com/owncloud/ocis/issues/1285) #### [Difference in response content of status.php and default capabilities](https://github.com/owncloud/ocis/issues/1286) @@ -1071,8 +1071,8 @@ Scenario Outline: A disabled user cannot use webdav - [apiAuth/cors.feature:185](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiAuth/cors.feature#L185) #### [App Passwords/Tokens for legacy WebDAV clients](https://github.com/owncloud/ocis/issues/197) -- [apiAuthWebDav/webDavDELETEAuth.feature:80](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiAuthWebDav/webDavDELETEAuth.feature#L80) -- [apiAuthWebDav/webDavDELETEAuth.feature:94](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiAuthWebDav/webDavDELETEAuth.feature#L94) +- [apiAuthWebDav/webDavDELETEAuth.feature:136](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiAuthWebDav/webDavDELETEAuth.feature#L136) +- [apiAuthWebDav/webDavDELETEAuth.feature:162](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiAuthWebDav/webDavDELETEAuth.feature#L162) #### [various sharing settings cannot be set](https://github.com/owncloud/ocis/issues/1328) - [apiCapabilities/capabilities.feature:8](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiCapabilities/capabilities.feature#L8) @@ -1374,5 +1374,12 @@ _ocs: api compatibility, return correct status code_ - [apiShareOperationsToShares2/shareAccessByID.feature:125](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiShareOperationsToShares2/shareAccessByID.feature#L125) ### [Content-type is not multipart/byteranges when downloading file with Range Header](https://github.com/owncloud/ocis/issues/2677) -- [apiWebdavOperations/downloadFile.feature:169](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavOperations/downloadFile.feature#L169) -- [apiWebdavOperations/downloadFile.feature:170](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavOperations/downloadFile.feature#L170) \ No newline at end of file +- [apiWebdavOperations/downloadFile.feature:169](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavOperations/downloadFile.feature#L169) +- [apiWebdavOperations/downloadFile.feature:170](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavOperations/downloadFile.feature#L170) + +#### [Creating a new folder which is a substring of Shares leads to Unknown Error](https://github.com/owncloud/ocis/issues/3033) +- [apiWebdavProperties1/createFileFolderWhenSharesExist.feature:29](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavProperties1/createFileFolderWhenSharesExist.feature#L29) +- [apiWebdavProperties1/createFileFolderWhenSharesExist.feature:32](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavProperties1/createFileFolderWhenSharesExist.feature#L32) +- [apiWebdavProperties1/createFileFolderWhenSharesExist.feature:45](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavProperties1/createFileFolderWhenSharesExist.feature#L45) +- [apiWebdavProperties1/createFileFolderWhenSharesExist.feature:48](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavProperties1/createFileFolderWhenSharesExist.feature#L48) +- [apiWebdavProperties1/createFileFolderWhenSharesExist.feature:60](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavProperties1/createFileFolderWhenSharesExist.feature#L60) \ No newline at end of file diff --git a/tests/acceptance/expected-failures-on-S3NG-storage.md b/tests/acceptance/expected-failures-on-S3NG-storage.md index 5991fa4bf8..d5547fa96f 100644 --- a/tests/acceptance/expected-failures-on-S3NG-storage.md +++ b/tests/acceptance/expected-failures-on-S3NG-storage.md @@ -929,26 +929,26 @@ API, search, favorites, config, capabilities, not existing endpoints, CORS and o #### [Trying to access another user's file gives http 403 instead of 404](https://github.com/owncloud/ocis/issues/2175) _ocdav: api compatibility, return correct status code_ -- [apiAuthWebDav/webDavDELETEAuth.feature:38](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiAuthWebDav/webDavDELETEAuth.feature#L38) Scenario: send DELETE requests to another user's webDav endpoints as normal user -- [apiAuthWebDav/webDavPROPFINDAuth.feature:39](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiAuthWebDav/webDavPROPFINDAuth.feature#L39) Scenario: send PROPFIND requests to another user's webDav endpoints as normal user -- [apiAuthWebDav/webDavPROPPATCHAuth.feature:40](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiAuthWebDav/webDavPROPPATCHAuth.feature#L40) Scenario: send PROPPATCH requests to another user's webDav endpoints as normal user -- [apiAuthWebDav/webDavMKCOLAuth.feature:36](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiAuthWebDav/webDavMKCOLAuth.feature#L36) Scenario: send MKCOL requests to another user's webDav endpoints as normal user +- [apiAuthWebDav/webDavDELETEAuth.feature:58](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiAuthWebDav/webDavDELETEAuth.feature#L58) Scenario: send DELETE requests to another user's webDav endpoints as normal user +- [apiAuthWebDav/webDavPROPFINDAuth.feature:57](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiAuthWebDav/webDavPROPFINDAuth.feature#L57) Scenario: send PROPFIND requests to another user's webDav endpoints as normal user +- [apiAuthWebDav/webDavPROPPATCHAuth.feature:58](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiAuthWebDav/webDavPROPPATCHAuth.feature#L58) Scenario: send PROPPATCH requests to another user's webDav endpoints as normal user +- [apiAuthWebDav/webDavMKCOLAuth.feature:54](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiAuthWebDav/webDavMKCOLAuth.feature#L54) Scenario: send MKCOL requests to another user's webDav endpoints as normal user #### [trying to lock file of another user gives http 200](https://github.com/owncloud/ocis/issues/2176) -- [apiAuthWebDav/webDavLOCKAuth.feature:40](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiAuthWebDav/webDavLOCKAuth.feature#L40) Scenario: send LOCK requests to another user's webDav endpoints as normal user +- [apiAuthWebDav/webDavLOCKAuth.feature:58](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiAuthWebDav/webDavLOCKAuth.feature#L58) Scenario: send LOCK requests to another user's webDav endpoints as normal user #### [Renaming a resource to banned name is allowed](https://github.com/owncloud/ocis/issues/1295) _ocdav: api compatibility, return correct status code_ -- [apiAuthWebDav/webDavMOVEAuth.feature:39](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiAuthWebDav/webDavMOVEAuth.feature#L39) Scenario: send MOVE requests to another user's webDav endpoints as normal user +- [apiAuthWebDav/webDavMOVEAuth.feature:57](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiAuthWebDav/webDavMOVEAuth.feature#L57) Scenario: send MOVE requests to another user's webDav endpoints as normal user #### [send POST requests to another user's webDav endpoints as normal user](https://github.com/owncloud/ocis/issues/1287) _ocdav: api compatibility, return correct status code_ -- [apiAuthWebDav/webDavPOSTAuth.feature:40](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiAuthWebDav/webDavPOSTAuth.feature#L40) Scenario: send POST requests to another user's webDav endpoints as normal user +- [apiAuthWebDav/webDavPOSTAuth.feature:58](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiAuthWebDav/webDavPOSTAuth.feature#L58) Scenario: send POST requests to another user's webDav endpoints as normal user #### [Using double slash in URL to access a folder gives 501 and other status codes](https://github.com/owncloud/ocis/issues/1667) -- [apiAuthWebDav/webDavSpecialURLs.feature:24](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiAuthWebDav/webDavSpecialURLs.feature#L24) -- [apiAuthWebDav/webDavSpecialURLs.feature:69](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiAuthWebDav/webDavSpecialURLs.feature#L69) -- [apiAuthWebDav/webDavSpecialURLs.feature:91](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiAuthWebDav/webDavSpecialURLs.feature#L91) +- [apiAuthWebDav/webDavSpecialURLs.feature:34](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiAuthWebDav/webDavSpecialURLs.feature#L34) +- [apiAuthWebDav/webDavSpecialURLs.feature:121](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiAuthWebDav/webDavSpecialURLs.feature#L121) +- [apiAuthWebDav/webDavSpecialURLs.feature:163](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiAuthWebDav/webDavSpecialURLs.feature#L163) #### [Default capabilities for normal user not same as in oC-core](https://github.com/owncloud/ocis/issues/1285) #### [Difference in response content of status.php and default capabilities](https://github.com/owncloud/ocis/issues/1286) @@ -1086,8 +1086,8 @@ Scenario Outline: A disabled user cannot use webdav - [apiAuth/cors.feature:185](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiAuth/cors.feature#L185) #### [App Passwords/Tokens for legacy WebDAV clients](https://github.com/owncloud/ocis/issues/197) -- [apiAuthWebDav/webDavDELETEAuth.feature:80](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiAuthWebDav/webDavDELETEAuth.feature#L80) -- [apiAuthWebDav/webDavDELETEAuth.feature:94](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiAuthWebDav/webDavDELETEAuth.feature#L94) +- [apiAuthWebDav/webDavDELETEAuth.feature:136](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiAuthWebDav/webDavDELETEAuth.feature#L136) +- [apiAuthWebDav/webDavDELETEAuth.feature:162](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiAuthWebDav/webDavDELETEAuth.feature#L162) #### [various sharing settings cannot be set](https://github.com/owncloud/ocis/issues/1328) - [apiCapabilities/capabilities.feature:8](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiCapabilities/capabilities.feature#L8) @@ -1374,5 +1374,12 @@ _ocs: api compatibility, return correct status code_ - [apiShareOperationsToShares2/shareAccessByID.feature:125](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiShareOperationsToShares2/shareAccessByID.feature#L125) ### [Content-type is not multipart/byteranges when downloading file with Range Header](https://github.com/owncloud/ocis/issues/2677) -- [apiWebdavOperations/downloadFile.feature:169](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavOperations/downloadFile.feature#L169) -- [apiWebdavOperations/downloadFile.feature:170](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavOperations/downloadFile.feature#L170) \ No newline at end of file +- [apiWebdavOperations/downloadFile.feature:169](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavOperations/downloadFile.feature#L169) +- [apiWebdavOperations/downloadFile.feature:170](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavOperations/downloadFile.feature#L170) + +#### [Creating a new folder which is a substring of Shares leads to Unknown Error](https://github.com/owncloud/ocis/issues/3033) +- [apiWebdavProperties1/createFileFolderWhenSharesExist.feature:29](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavProperties1/createFileFolderWhenSharesExist.feature#L29) +- [apiWebdavProperties1/createFileFolderWhenSharesExist.feature:32](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavProperties1/createFileFolderWhenSharesExist.feature#L32) +- [apiWebdavProperties1/createFileFolderWhenSharesExist.feature:45](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavProperties1/createFileFolderWhenSharesExist.feature#L45) +- [apiWebdavProperties1/createFileFolderWhenSharesExist.feature:48](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavProperties1/createFileFolderWhenSharesExist.feature#L48) +- [apiWebdavProperties1/createFileFolderWhenSharesExist.feature:60](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavProperties1/createFileFolderWhenSharesExist.feature#L60) \ No newline at end of file From ac0f51d0df58f8e45caf93ad8f1dd53465ceb7f6 Mon Sep 17 00:00:00 2001 From: David Christofas Date: Wed, 26 Jan 2022 17:04:29 +0100 Subject: [PATCH 57/58] fix test failures --- internal/grpc/services/gateway/authprovider.go | 15 ++++++++------- .../owncloud/ocdav/spacelookup/spacelookup.go | 6 +++++- pkg/auth/scope/publicshare.go | 3 +++ 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/internal/grpc/services/gateway/authprovider.go b/internal/grpc/services/gateway/authprovider.go index d716f8a7b7..ed8357ab7f 100644 --- a/internal/grpc/services/gateway/authprovider.go +++ b/internal/grpc/services/gateway/authprovider.go @@ -21,19 +21,24 @@ package gateway import ( "context" "fmt" + "strings" authpb "github.com/cs3org/go-cs3apis/cs3/auth/provider/v1beta1" registry "github.com/cs3org/go-cs3apis/cs3/auth/registry/v1beta1" gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1" userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" + collaboration "github.com/cs3org/go-cs3apis/cs3/sharing/collaboration/v1beta1" + link "github.com/cs3org/go-cs3apis/cs3/sharing/link/v1beta1" storageprovider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/cs3org/reva/pkg/appctx" + "github.com/cs3org/reva/pkg/auth/scope" ctxpkg "github.com/cs3org/reva/pkg/ctx" "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/rgrpc/status" "github.com/cs3org/reva/pkg/rgrpc/todo/pool" "github.com/cs3org/reva/pkg/sharedconf" + "github.com/cs3org/reva/pkg/utils" "github.com/pkg/errors" "google.golang.org/grpc/metadata" ) @@ -114,18 +119,16 @@ func (s *svc) Authenticate(ctx context.Context, req *gateway.AuthenticateRequest ctx = ctxpkg.ContextSetUser(ctx, res.User) ctx = metadata.AppendToOutgoingContext(ctx, ctxpkg.TokenHeader, token) - // Commenting out as the token size can get too big - // For now, we'll try to resolve all resources on every request // TODO(ishank011): Add a cache for these - /* scope, err := s.expandScopes(ctx, res.TokenScope) + scope, err := s.expandScopes(ctx, res.TokenScope) if err != nil { err = errors.Wrap(err, "authsvc: error expanding token scope") return &gateway.AuthenticateResponse{ Status: status.NewUnauthenticated(ctx, err, "error expanding access token scope"), }, nil } - */ - scope := res.TokenScope + + // scope := res.TokenScope token, err = s.tokenmgr.MintToken(ctx, &u, scope) if err != nil { @@ -234,7 +237,6 @@ func (s *svc) findAuthProvider(ctx context.Context, authType string) (authpb.Pro return nil, errtypes.InternalError("gateway: error finding an auth provider for type: " + authType) } -/* func (s *svc) expandScopes(ctx context.Context, scopeMap map[string]*authpb.Scope) (map[string]*authpb.Scope, error) { log := appctx.GetLogger(ctx) newMap := make(map[string]*authpb.Scope) @@ -305,4 +307,3 @@ func (s *svc) statAndAddResource(ctx context.Context, r *storageprovider.Resourc return scope.AddResourceInfoScope(statResponse.Info, role, scopeMap) } -*/ diff --git a/internal/http/services/owncloud/ocdav/spacelookup/spacelookup.go b/internal/http/services/owncloud/ocdav/spacelookup/spacelookup.go index 3be319bec2..b97b18aee0 100644 --- a/internal/http/services/owncloud/ocdav/spacelookup/spacelookup.go +++ b/internal/http/services/owncloud/ocdav/spacelookup/spacelookup.go @@ -57,7 +57,11 @@ func LookUpStorageSpaceForPath(ctx context.Context, client gateway.GatewayAPICli lSSRes, err := client.ListStorageSpaces(ctx, lSSReq) if err != nil || lSSRes.Status.Code != rpc.Code_CODE_OK { - return nil, lSSRes.Status, err + status := status.NewStatusFromErrType(ctx, "failed to lookup storage spaces", err) + if lSSRes != nil { + status = lSSRes.Status + } + return nil, status, err } switch len(lSSRes.StorageSpaces) { case 0: diff --git a/pkg/auth/scope/publicshare.go b/pkg/auth/scope/publicshare.go index bd52bbfea3..0ca65d8729 100644 --- a/pkg/auth/scope/publicshare.go +++ b/pkg/auth/scope/publicshare.go @@ -27,6 +27,7 @@ import ( authpb "github.com/cs3org/go-cs3apis/cs3/auth/provider/v1beta1" gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1" userv1beta1 "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" + permissionsv1beta1 "github.com/cs3org/go-cs3apis/cs3/permissions/v1beta1" link "github.com/cs3org/go-cs3apis/cs3/sharing/link/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" registry "github.com/cs3org/go-cs3apis/cs3/storage/registry/v1beta1" @@ -81,6 +82,8 @@ func publicshareScope(ctx context.Context, scope *authpb.Scope, resource interfa return checkStorageRef(ctx, &share, &provider.Reference{ResourceId: v.ResourceInfo.Id}), nil case *gateway.OpenInAppRequest: return checkStorageRef(ctx, &share, v.GetRef()), nil + case *permissionsv1beta1.CheckPermissionRequest: + return true, nil // Editor role // need to return appropriate status codes in the ocs/ocdav layers. From 0d53a05daf9c3bc40083b1e19483479a0a555707 Mon Sep 17 00:00:00 2001 From: David Christofas Date: Thu, 27 Jan 2022 15:02:37 +0100 Subject: [PATCH 58/58] fix integration tests --- tests/integration/grpc/fixtures/gateway-sharded.toml | 1 + tests/integration/grpc/fixtures/gateway-static.toml | 1 + .../grpc/fixtures/storageprovider-ocis.toml | 3 ++- tests/integration/grpc/gateway_storageprovider_test.go | 10 ++++++---- 4 files changed, 10 insertions(+), 5 deletions(-) diff --git a/tests/integration/grpc/fixtures/gateway-sharded.toml b/tests/integration/grpc/fixtures/gateway-sharded.toml index bd4c1298e0..a74b3c4186 100644 --- a/tests/integration/grpc/fixtures/gateway-sharded.toml +++ b/tests/integration/grpc/fixtures/gateway-sharded.toml @@ -9,6 +9,7 @@ address = "{{grpc_address}}" # registries storageregistrysvc = "{{grpc_address}}" stat_cache_ttl = 1 +permissionssvc = "{{permissions_address}}" [grpc.services.authregistry] driver = "static" diff --git a/tests/integration/grpc/fixtures/gateway-static.toml b/tests/integration/grpc/fixtures/gateway-static.toml index df8a7abecf..b8d1681e14 100644 --- a/tests/integration/grpc/fixtures/gateway-static.toml +++ b/tests/integration/grpc/fixtures/gateway-static.toml @@ -9,6 +9,7 @@ address = "{{grpc_address}}" # registries storageregistrysvc = "{{grpc_address}}" stat_cache_ttl = 1 +permissionssvc = "{{permissions_address}}" [grpc.services.authregistry] driver = "static" diff --git a/tests/integration/grpc/fixtures/storageprovider-ocis.toml b/tests/integration/grpc/fixtures/storageprovider-ocis.toml index e5c52002d6..49c0dd1af5 100644 --- a/tests/integration/grpc/fixtures/storageprovider-ocis.toml +++ b/tests/integration/grpc/fixtures/storageprovider-ocis.toml @@ -9,4 +9,5 @@ root = "{{root}}/storage" treetime_accounting = true treesize_accounting = true enable_home = true -userprovidersvc = "localhost:18000" \ No newline at end of file +userprovidersvc = "localhost:18000" +gateway_addr = "{{gateway_address}}" \ No newline at end of file diff --git a/tests/integration/grpc/gateway_storageprovider_test.go b/tests/integration/grpc/gateway_storageprovider_test.go index 109875439e..2fac29bdc1 100644 --- a/tests/integration/grpc/gateway_storageprovider_test.go +++ b/tests/integration/grpc/gateway_storageprovider_test.go @@ -91,10 +91,11 @@ var _ = Describe("gateway", func() { BeforeEach(func() { dependencies = map[string]string{ - "gateway": "gateway.toml", - "users": "userprovider-json.toml", - "storage": "storageprovider-ocis.toml", - "storage2": "storageprovider-ocis.toml", + "gateway": "gateway.toml", + "users": "userprovider-json.toml", + "storage": "storageprovider-ocis.toml", + "storage2": "storageprovider-ocis.toml", + "permissions": "permissions-ocis-ci.toml", } }) @@ -167,6 +168,7 @@ var _ = Describe("gateway", func() { "homestorage": "storageprovider-ocis.toml", "storage": "storageprovider-ocis.toml", "storage2": "storageprovider-ocis.toml", + "permissions": "permissions-ocis-ci.toml", } })