Merge branch 'master' into edge

.dockerignore

@@ -1,7 +1,6 @@
 .github
 changelog
 docs
-examples
 grpc-tests
 tests
 tools

.drone.star

@@ -106,6 +106,8 @@ def main(ctx):
         litmusOcisOldWebdav(),
         litmusOcisNewWebdav(),
         litmusOcisSpacesDav(),
+        # virtual views don't work on edge at the moment
+        #virtualViews(),
     ] + ocisIntegrationTests(6) + s3ngIntegrationTests(12)
 
 def buildAndPublishDocker():
@@ -228,6 +230,26 @@ def buildAndPublishDocker():
                     ],
                 },
             },
+            {
+                "name": "publish-docker-revad-ceph-latest",
+                "pull": "always",
+                "image": "plugins/docker",
+                "settings": {
+                    "repo": "cs3org/revad",
+                    "tags": "latest-ceph",
+                    "dockerfile": "Dockerfile.revad-ceph",
+                    "username": {
+                        "from_secret": "dockerhub_username",
+                    },
+                    "password": {
+                        "from_secret": "dockerhub_password",
+                    },
+                    "custom_dns": [
+                        "128.142.17.5",
+                        "128.142.16.5",
+                    ],
+                },
+            },
         ],
     }
 
@@ -481,6 +503,84 @@ def release():
                     ],
                 },
             },
+            {
+                "name": "docker-revad-ceph-tag",
+                "pull": "always",
+                "image": "plugins/docker",
+                "settings": {
+                    "repo": "cs3org/revad",
+                    "tags": "${DRONE_TAG}-ceph",
+                    "dockerfile": "Dockerfile.revad-ceph",
+                    "username": {
+                        "from_secret": "dockerhub_username",
+                    },
+                    "password": {
+                        "from_secret": "dockerhub_password",
+                    },
+                    "custom_dns": [
+                        "128.142.17.5",
+                        "128.142.16.5",
+                    ],
+                },
+            },
+        ],
+        "depends_on": ["changelog"],
+    }
+
+def virtualViews():
+    return {
+        "kind": "pipeline",
+        "type": "docker",
+        "name": "virtual-views",
+        "platform": {
+            "os": "linux",
+            "arch": "amd64",
+        },
+        "trigger": {
+            "event": {
+                "include": [
+                    "pull_request",
+                    "tag",
+                ],
+            },
+        },
+        "steps": [
+            makeStep("build-ci"),
+            {
+                "name": "revad-services",
+                "image": "registry.cern.ch/docker.io/library/golang:1.17",
+                "detach": True,
+                "commands": [
+                    "cd /drone/src/tests/oc-integration-tests/drone/",
+                    "/drone/src/cmd/revad/revad -c frontend-global.toml &",
+                    "/drone/src/cmd/revad/revad -c gateway.toml &",
+                    "/drone/src/cmd/revad/revad -c storage-home-ocis.toml &",
+                    "/drone/src/cmd/revad/revad -c storage-local-1.toml &",
+                    "/drone/src/cmd/revad/revad -c storage-local-2.toml &",
+                    "/drone/src/cmd/revad/revad -c users.toml",
+                ],
+            },
+            cloneOc10TestReposStep(),
+            {
+                "name": "oC10APIAcceptanceTestsOcisStorage",
+                "image": "registry.cern.ch/docker.io/owncloudci/php:7.4",
+                "commands": [
+                    "cd /drone/src",
+                    "make test-acceptance-api",
+                ],
+                "environment": {
+                    "PATH_TO_CORE": "/drone/src/tmp/testrunner",
+                    "TEST_SERVER_URL": "http://revad-services:20180",
+                    "OCIS_REVA_DATA_ROOT": "/drone/src/tmp/reva/data/",
+                    "DELETE_USER_DATA_CMD": "rm -rf /drone/src/tmp/reva/data/nodes/root/* /drone/src/tmp/reva/data/nodes/*-*-*-* /drone/src/tmp/reva/data/blobs/*",
+                    "STORAGE_DRIVER": "OCIS",
+                    "SKELETON_DIR": "/drone/src/tmp/testing/data/apiSkeleton",
+                    "TEST_REVA": "true",
+                    "REGULAR_USER_PASSWORD": "relativity",
+                    "SEND_SCENARIO_LINE_REFERENCES": "true",
+                    "BEHAT_SUITE": "apiVirtualViews",
+                },
+            },
         ],
         "depends_on": ["changelog"],
     }
@@ -516,6 +616,7 @@ def litmusOcisOldWebdav():
                     "/drone/src/cmd/revad/revad -c storage-shares.toml &",
                     "/drone/src/cmd/revad/revad -c storage-publiclink.toml &",
                     "/drone/src/cmd/revad/revad -c shares.toml &",
+                    "/drone/src/cmd/revad/revad -c permissions-ocis-ci.toml &",
                     "/drone/src/cmd/revad/revad -c users.toml",
                 ],
             },
@@ -571,6 +672,7 @@ def litmusOcisNewWebdav():
                     "/drone/src/cmd/revad/revad -c storage-shares.toml &",
                     "/drone/src/cmd/revad/revad -c storage-publiclink.toml &",
                     "/drone/src/cmd/revad/revad -c shares.toml &",
+                    "/drone/src/cmd/revad/revad -c permissions-ocis-ci.toml &",
                     "/drone/src/cmd/revad/revad -c users.toml",
                 ],
             },
@@ -627,6 +729,7 @@ def litmusOcisSpacesDav():
                     "/drone/src/cmd/revad/revad -c storage-shares.toml &",
                     "/drone/src/cmd/revad/revad -c storage-publiclink.toml &",
                     "/drone/src/cmd/revad/revad -c shares.toml &",
+                    "/drone/src/cmd/revad/revad -c permissions-ocis-ci.toml &",
                     "/drone/src/cmd/revad/revad -c users.toml",
                 ],
             },
@@ -695,6 +798,7 @@ def ocisIntegrationTests(parallelRuns, skipExceptParts = []):
                             "/drone/src/cmd/revad/revad -c machine-auth.toml &",
                             "/drone/src/cmd/revad/revad -c storage-users-ocis.toml &",
                             "/drone/src/cmd/revad/revad -c storage-publiclink.toml &",
+                            "/drone/src/cmd/revad/revad -c permissions-ocis-ci.toml &",
                             "/drone/src/cmd/revad/revad -c ldap-users.toml",
                         ],
                     },
@@ -771,6 +875,7 @@ def s3ngIntegrationTests(parallelRuns, skipExceptParts = []):
                             "/drone/src/cmd/revad/revad -c storage-publiclink.toml &",
                             "/drone/src/cmd/revad/revad -c storage-shares.toml &",
                             "/drone/src/cmd/revad/revad -c ldap-users.toml &",
+                            "/drone/src/cmd/revad/revad -c permissions-ocis-ci.toml &",
                             "/drone/src/cmd/revad/revad -c machine-auth.toml",
                         ],
                     },

Dockerfile.revad-ceph

@@ -0,0 +1,52 @@
+# Copyright 2018-2021 CERN
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# In applying this license, CERN does not waive the privileges and immunities
+# granted to it by virtue of its status as an Intergovernmental Organization
+# or submit itself to any jurisdiction.
+
+FROM ceph/daemon-base
+
+RUN dnf update -y && dnf install -y \
+  git \
+  gcc \
+  make \
+  libcephfs-devel \
+  librbd-devel \
+  librados-devel
+
+ADD https://golang.org/dl/go1.16.4.linux-amd64.tar.gz \
+  go1.16.4.linux-amd64.tar.gz
+
+RUN rm -rf /usr/local/go && \
+  tar -C /usr/local -xzf go1.16.4.linux-amd64.tar.gz && \
+  rm go1.16.4.linux-amd64.tar.gz
+
+ENV PATH /go/bin:/usr/local/go/bin:$PATH
+ENV GOPATH /go
+
+WORKDIR /go/src/github/cs3org/reva
+COPY . .
+RUN mkdir -p /go/bin \
+    make build-revad-cephfs-docker && \
+    cp /go/src/github/cs3org/reva/cmd/revad/revad /usr/bin/revad
+
+RUN cp -r examples/ceph /etc/
+
+RUN mkdir -p /etc/revad/ && echo "" > /etc/revad/revad.toml
+
+EXPOSE 9999 10000
+
+ENTRYPOINT [ "/usr/bin/revad" ]
+CMD [ "-c", "/etc/revad/revad.toml", "-p", "/var/run/revad.pid" ]

Makefile

@@ -30,17 +30,19 @@ off:
 imports: off
 	`go env GOPATH`/bin/goimports -w tools pkg internal cmd
 
-build: imports test-go-version
-	gofmt -s -w .
-	go build -ldflags ${BUILD_FLAGS} -o ./cmd/revad/revad ./cmd/revad
-	go build -ldflags ${BUILD_FLAGS} -o ./cmd/reva/reva ./cmd/reva
+build: build-revad build-reva test-go-version
+
+build-cephfs: build-revad-cephfs build-reva
 
 tidy:
 	go mod tidy
 
 build-revad: imports
 	go build -ldflags ${BUILD_FLAGS} -o ./cmd/revad/revad ./cmd/revad
 
+build-revad-cephfs: imports
+	go build -ldflags ${BUILD_FLAGS} -tags ceph -o ./cmd/revad/revad ./cmd/revad
+
 build-reva: imports
 	go build -ldflags ${BUILD_FLAGS} -o ./cmd/reva/reva ./cmd/reva
 
@@ -104,6 +106,8 @@ ci: build-ci test  lint-ci
 # to be run in Docker build
 build-revad-docker: off
 	go build -ldflags ${BUILD_FLAGS} -o ./cmd/revad/revad ./cmd/revad
+build-revad-cephfs-docker: off
+	go build -ldflags ${BUILD_FLAGS} -tags ceph -o ./cmd/revad/revad ./cmd/revad
 build-reva-docker: off
 	go build -ldflags ${BUILD_FLAGS} -o ./cmd/reva/reva ./cmd/reva
 clean:

changelog/unreleased/cephfs-driver.md

@@ -0,0 +1,3 @@
+Enhancement: Reva CephFS module v0.2.1
+
+https://github.com/cs3org/reva/pull/1209

changelog/unreleased/cs3-permissions-service.md

@@ -0,0 +1,5 @@
+Enhancement: Use CS3 permissions API
+
+Added calls to the CS3 permissions API to the decomposedfs in order to check the user permissions.
+
+https://github.com/cs3org/reva/pull/2341

cmd/revad/runtime/loader.go

@@ -38,6 +38,7 @@ import (
 	_ "github.com/cs3org/reva/pkg/ocm/invite/manager/loader"
 	_ "github.com/cs3org/reva/pkg/ocm/provider/authorizer/loader"
 	_ "github.com/cs3org/reva/pkg/ocm/share/manager/loader"
+	_ "github.com/cs3org/reva/pkg/permission/manager/loader"
 	_ "github.com/cs3org/reva/pkg/publicshare/manager/loader"
 	_ "github.com/cs3org/reva/pkg/rhttp/datatx/manager/loader"
 	_ "github.com/cs3org/reva/pkg/share/cache/loader"

docs/content/en/docs/config/packages/storage/fs/cephfs/_index.md

@@ -0,0 +1,26 @@
+---
+title: "cephfs"
+linkTitle: "cephfs"
+weight: 10
+description: >
+  Configuration for the cephfs service
+---
+
+# _struct: config_
+
+{{% dir name="root" type="string" default="/var/tmp/reva/" %}}
+Path of root directory for user storage. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/storage/fs/cephfs/cephfs.go#L34)
+{{< highlight toml >}}
+[storage.fs.cephfs]
+root = "/var/tmp/reva/"
+{{< /highlight >}}
+{{% /dir %}}
+
+{{% dir name="share_folder" type="string" default="/MyShares" %}}
+Path for storing share references. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/storage/fs/cephfs/cephfs.go#L35)
+{{< highlight toml >}}
+[storage.fs.cephfs]
+share_folder = "/MyShares"
+{{< /highlight >}}
+{{% /dir %}}
+

examples/ceph/ceph.conf

@@ -0,0 +1,3 @@
+[global]
+  fsid = '8aaa35c4-75dc-42e5-a812-cbc1cdfd3323'
+  mon_host = '[v2:188.184.96.178:3300/0,v1:188.184.96.178:6789/0] [v2:188.185.88.76:3300/0,v1:188.185.88.76:6789/0] [v2:188.185.126.6:3300/0,v1:188.185.126.6:6789/0]'

examples/ceph/keyring

@@ -0,0 +1,2 @@
+[client.admin]
+  key = 'AQAu88Fg5iekGhAAeVP0Td05PuybytuRJgBRqA=='

go.mod

@@ -3,21 +3,23 @@ module github.com/cs3org/reva
 require (
 	bou.ke/monkey v1.0.2
 	contrib.go.opencensus.io/exporter/prometheus v0.4.0
-	github.com/BurntSushi/toml v0.4.1
+	github.com/BurntSushi/toml v1.0.0
 	github.com/Masterminds/goutils v1.1.1 // indirect
 	github.com/Masterminds/semver v1.5.0 // indirect
 	github.com/Masterminds/sprig v2.22.0+incompatible
 	github.com/ReneKroon/ttlcache/v2 v2.11.0
 	github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect
-	github.com/aws/aws-sdk-go v1.42.27
+	github.com/aws/aws-sdk-go v1.42.39
 	github.com/beevik/etree v1.1.0
 	github.com/bluele/gcache v0.0.2
 	github.com/c-bata/go-prompt v0.2.5
+	github.com/ceph/go-ceph v0.13.0
 	github.com/cheggaaa/pb v1.0.29
 	github.com/coreos/go-oidc v2.2.1+incompatible
 	github.com/cs3org/cato v0.0.0-20200828125504-e418fc54dd5e
-	github.com/cs3org/go-cs3apis v0.0.0-20211214102047-7ce3134d7bf8
+	github.com/cs3org/go-cs3apis v0.0.0-20211214102128-4e8745ab1654
 	github.com/cubewise-code/go-mime v0.0.0-20200519001935-8c5762b177d8
+	github.com/dgraph-io/ristretto v0.1.0
 	github.com/eventials/go-tus v0.0.0-20200718001131-45c7ec8f5d59
 	github.com/gdexlab/go-render v1.0.1
 	github.com/go-chi/chi/v5 v5.0.7
@@ -28,25 +30,26 @@ require (
 	github.com/golang-jwt/jwt v3.2.2+incompatible
 	github.com/golang/protobuf v1.5.2
 	github.com/gomodule/redigo v1.8.8
-	github.com/google/go-cmp v0.5.6
+	github.com/google/go-cmp v0.5.7
 	github.com/google/go-github v17.0.0+incompatible
 	github.com/google/go-querystring v1.1.0 // indirect
 	github.com/google/uuid v1.3.0
 	github.com/grpc-ecosystem/go-grpc-middleware v1.3.0
-	github.com/hashicorp/go-hclog v1.0.0
+	github.com/hashicorp/go-hclog v1.1.0
 	github.com/hashicorp/go-plugin v1.4.3
 	github.com/huandu/xstrings v1.3.2 // indirect
 	github.com/imdario/mergo v0.3.12 // indirect
 	github.com/jedib0t/go-pretty v4.3.0+incompatible
 	github.com/juliangruber/go-intersect v1.1.0
 	github.com/mattn/go-sqlite3 v1.14.10
+	github.com/maxymania/go-system v0.0.0-20170110133659-647cc364bf0b
 	github.com/mileusna/useragent v1.0.2
-	github.com/minio/minio-go/v7 v7.0.20
+	github.com/minio/minio-go/v7 v7.0.21
 	github.com/mitchellh/copystructure v1.2.0 // indirect
 	github.com/mitchellh/go-testing-interface v1.14.1 // indirect
 	github.com/mitchellh/mapstructure v1.4.3
 	github.com/onsi/ginkgo v1.16.5
-	github.com/onsi/gomega v1.17.0
+	github.com/onsi/gomega v1.18.0
 	github.com/pkg/errors v0.9.1
 	github.com/pkg/xattr v0.4.4
 	github.com/pquerna/cachecontrol v0.1.0 // indirect
@@ -71,7 +74,8 @@ require (
 	go.opentelemetry.io/otel/trace v1.3.0
 	golang.org/x/crypto v0.0.0-20211215165025-cf75a172585e
 	golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1
-	golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359
+	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
+	golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e
 	golang.org/x/term v0.0.0-20210916214954-140adaaadfaf
 	google.golang.org/genproto v0.0.0-20211021150943-2b146023228c
 	google.golang.org/grpc v1.42.0